1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for the VirtIO PCI interface. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.c 268010 2014-06-29 00:37:59Z bryanv $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/malloc.h>
38
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus.h>
42#include <sys/rman.h>
43
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46
47#include <dev/virtio/virtio.h>
48#include <dev/virtio/virtqueue.h>
49#include <dev/virtio/pci/virtio_pci.h>
50
51#include "virtio_bus_if.h"
52#include "virtio_if.h"
53
54struct vtpci_interrupt {
55 struct resource *vti_irq;
56 int vti_rid;
57 void *vti_handler;
58};
59
60struct vtpci_virtqueue {
61 struct virtqueue *vtv_vq;
62 int vtv_no_intr;
63};
64
65struct vtpci_softc {
66 device_t vtpci_dev;
67 struct resource *vtpci_res;
68 struct resource *vtpci_msix_res;
69 uint64_t vtpci_features;
70 uint32_t vtpci_flags;
71#define VTPCI_FLAG_NO_MSI 0x0001
72#define VTPCI_FLAG_NO_MSIX 0x0002
73#define VTPCI_FLAG_LEGACY 0x1000
74#define VTPCI_FLAG_MSI 0x2000
75#define VTPCI_FLAG_MSIX 0x4000
76#define VTPCI_FLAG_SHARED_MSIX 0x8000
77#define VTPCI_FLAG_ITYPE_MASK 0xF000
78
79 /* This "bus" will only ever have one child. */
80 device_t vtpci_child_dev;
81 struct virtio_feature_desc *vtpci_child_feat_desc;
82
83 int vtpci_nvqs;
84 struct vtpci_virtqueue *vtpci_vqs;
85
86 /*
87 * Ideally, each virtqueue that the driver provides a callback for will
88 * receive its own MSIX vector. If there are not sufficient vectors
89 * available, then attempt to have all the VQs share one vector. For
90 * MSIX, the configuration changed notifications must be on their own
91 * vector.
92 *
93 * If MSIX is not available, we will attempt to have the whole device
94 * share one MSI vector, and then, finally, one legacy interrupt.
95 */
96 struct vtpci_interrupt vtpci_device_interrupt;
97 struct vtpci_interrupt *vtpci_msix_vq_interrupts;
98 int vtpci_nmsix_resources;
99};
100
101static int vtpci_probe(device_t);
102static int vtpci_attach(device_t);
103static int vtpci_detach(device_t);
104static int vtpci_suspend(device_t);
105static int vtpci_resume(device_t);
106static int vtpci_shutdown(device_t);
107static void vtpci_driver_added(device_t, driver_t *);
108static void vtpci_child_detached(device_t, device_t);
109static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
110static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
111
112static uint64_t vtpci_negotiate_features(device_t, uint64_t);
113static int vtpci_with_feature(device_t, uint64_t);
114static int vtpci_alloc_virtqueues(device_t, int, int,
115 struct vq_alloc_info *);
116static int vtpci_setup_intr(device_t, enum intr_type);
117static void vtpci_stop(device_t);
118static int vtpci_reinit(device_t, uint64_t);
119static void vtpci_reinit_complete(device_t);
120static void vtpci_notify_virtqueue(device_t, uint16_t);
121static uint8_t vtpci_get_status(device_t);
122static void vtpci_set_status(device_t, uint8_t);
123static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
124static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
125
126static void vtpci_describe_features(struct vtpci_softc *, const char *,
127 uint64_t);
128static void vtpci_probe_and_attach_child(struct vtpci_softc *);
129
130static int vtpci_alloc_msix(struct vtpci_softc *, int);
131static int vtpci_alloc_msi(struct vtpci_softc *);
132static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
133static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
134static int vtpci_alloc_intr_msi(struct vtpci_softc *);
135static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
136static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
137 struct vtpci_interrupt *);
138static int vtpci_alloc_intr_resources(struct vtpci_softc *);
139
140static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
141 enum intr_type);
142static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
143 enum intr_type);
144static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
145 enum intr_type);
146static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
147
148static int vtpci_register_msix_vector(struct vtpci_softc *, int,
149 struct vtpci_interrupt *);
150static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
151static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
152
153static void vtpci_free_interrupt(struct vtpci_softc *,
154 struct vtpci_interrupt *);
155static void vtpci_free_interrupts(struct vtpci_softc *);
156static void vtpci_free_virtqueues(struct vtpci_softc *);
157static void vtpci_release_child_resources(struct vtpci_softc *);
158static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
159static void vtpci_reset(struct vtpci_softc *);
160
161static void vtpci_select_virtqueue(struct vtpci_softc *, int);
162
163static void vtpci_legacy_intr(void *);
164static int vtpci_vq_shared_intr_filter(void *);
165static void vtpci_vq_shared_intr(void *);
166static int vtpci_vq_intr_filter(void *);
167static void vtpci_vq_intr(void *);
168static void vtpci_config_intr(void *);
169
170#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
171
172#define VIRTIO_PCI_CONFIG(_sc) \
173 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
174
175/*
176 * I/O port read/write wrappers.
177 */
178#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
179#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
180#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
181#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
182#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
183#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
184
185/* Tunables. */
186static int vtpci_disable_msix = 0;
187TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
188
189static device_method_t vtpci_methods[] = {
190 /* Device interface. */
191 DEVMETHOD(device_probe, vtpci_probe),
192 DEVMETHOD(device_attach, vtpci_attach),
193 DEVMETHOD(device_detach, vtpci_detach),
194 DEVMETHOD(device_suspend, vtpci_suspend),
195 DEVMETHOD(device_resume, vtpci_resume),
196 DEVMETHOD(device_shutdown, vtpci_shutdown),
197
198 /* Bus interface. */
199 DEVMETHOD(bus_driver_added, vtpci_driver_added),
200 DEVMETHOD(bus_child_detached, vtpci_child_detached),
201 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
202 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
203
204 /* VirtIO bus interface. */
205 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
206 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
207 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
208 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
209 DEVMETHOD(virtio_bus_stop, vtpci_stop),
210 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
211 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
212 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
213 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
214 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
215
216 DEVMETHOD_END
217};
218
219static driver_t vtpci_driver = {
220 "virtio_pci",
221 vtpci_methods,
222 sizeof(struct vtpci_softc)
223};
224
225devclass_t vtpci_devclass;
226
227DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
228MODULE_VERSION(virtio_pci, 1);
229MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
230MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
231
232static int
233vtpci_probe(device_t dev)
234{
235 char desc[36];
236 const char *name;
237
238 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
239 return (ENXIO);
240
241 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
242 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
243 return (ENXIO);
244
245 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
246 return (ENXIO);
247
248 name = virtio_device_name(pci_get_subdevice(dev));
249 if (name == NULL)
250 name = "Unknown";
251
252 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
253 device_set_desc_copy(dev, desc);
254
255 return (BUS_PROBE_DEFAULT);
256}
257
258static int
259vtpci_attach(device_t dev)
260{
261 struct vtpci_softc *sc;
262 device_t child;
263 int rid;
264
265 sc = device_get_softc(dev);
266 sc->vtpci_dev = dev;
267
268 pci_enable_busmaster(dev);
269
270 rid = PCIR_BAR(0);
271 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
272 RF_ACTIVE);
273 if (sc->vtpci_res == NULL) {
274 device_printf(dev, "cannot map I/O space\n");
275 return (ENXIO);
276 }
277
278 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
279 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
280
281 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
282 rid = PCIR_BAR(1);
283 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
284 SYS_RES_MEMORY, &rid, RF_ACTIVE);
285 }
286
287 if (sc->vtpci_msix_res == NULL)
288 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
289
290 vtpci_reset(sc);
291
292 /* Tell the host we've noticed this device. */
293 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
294
295 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
296 device_printf(dev, "cannot create child device\n");
297 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
298 vtpci_detach(dev);
299 return (ENOMEM);
300 }
301
302 sc->vtpci_child_dev = child;
303 vtpci_probe_and_attach_child(sc);
304
305 return (0);
306}
307
308static int
309vtpci_detach(device_t dev)
310{
311 struct vtpci_softc *sc;
312 device_t child;
313 int error;
314
315 sc = device_get_softc(dev);
316
317 if ((child = sc->vtpci_child_dev) != NULL) {
318 error = device_delete_child(dev, child);
319 if (error)
320 return (error);
321 sc->vtpci_child_dev = NULL;
322 }
323
324 vtpci_reset(sc);
325
326 if (sc->vtpci_msix_res != NULL) {
327 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
328 sc->vtpci_msix_res);
329 sc->vtpci_msix_res = NULL;
330 }
331
332 if (sc->vtpci_res != NULL) {
333 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
334 sc->vtpci_res);
335 sc->vtpci_res = NULL;
336 }
337
338 return (0);
339}
340
341static int
342vtpci_suspend(device_t dev)
343{
344
345 return (bus_generic_suspend(dev));
346}
347
348static int
349vtpci_resume(device_t dev)
350{
351
352 return (bus_generic_resume(dev));
353}
354
355static int
356vtpci_shutdown(device_t dev)
357{
358
359 (void) bus_generic_shutdown(dev);
360 /* Forcibly stop the host device. */
361 vtpci_stop(dev);
362
363 return (0);
364}
365
366static void
367vtpci_driver_added(device_t dev, driver_t *driver)
368{
369 struct vtpci_softc *sc;
370
371 sc = device_get_softc(dev);
372
373 vtpci_probe_and_attach_child(sc);
374}
375
376static void
377vtpci_child_detached(device_t dev, device_t child)
378{
379 struct vtpci_softc *sc;
380
381 sc = device_get_softc(dev);
382
383 vtpci_reset(sc);
384 vtpci_release_child_resources(sc);
385}
386
387static int
388vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
389{
390 struct vtpci_softc *sc;
391
392 sc = device_get_softc(dev);
393
394 if (sc->vtpci_child_dev != child)
395 return (ENOENT);
396
397 switch (index) {
398 case VIRTIO_IVAR_DEVTYPE:
399 case VIRTIO_IVAR_SUBDEVICE:
400 *result = pci_get_subdevice(dev);
401 break;
402 case VIRTIO_IVAR_VENDOR:
403 *result = pci_get_vendor(dev);
404 break;
405 case VIRTIO_IVAR_DEVICE:
406 *result = pci_get_device(dev);
407 break;
408 case VIRTIO_IVAR_SUBVENDOR:
409 *result = pci_get_subdevice(dev);
410 break;
411 default:
412 return (ENOENT);
413 }
414
415 return (0);
416}
417
418static int
419vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
420{
421 struct vtpci_softc *sc;
422
423 sc = device_get_softc(dev);
424
425 if (sc->vtpci_child_dev != child)
426 return (ENOENT);
427
428 switch (index) {
429 case VIRTIO_IVAR_FEATURE_DESC:
430 sc->vtpci_child_feat_desc = (void *) value;
431 break;
432 default:
433 return (ENOENT);
434 }
435
436 return (0);
437}
438
439static uint64_t
440vtpci_negotiate_features(device_t dev, uint64_t child_features)
441{
442 struct vtpci_softc *sc;
443 uint64_t host_features, features;
444
445 sc = device_get_softc(dev);
446
447 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
448 vtpci_describe_features(sc, "host", host_features);
449
450 /*
451 * Limit negotiated features to what the driver, virtqueue, and
452 * host all support.
453 */
454 features = host_features & child_features;
455 features = virtqueue_filter_features(features);
456 sc->vtpci_features = features;
457
458 vtpci_describe_features(sc, "negotiated", features);
459 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
460
461 return (features);
462}
463
464static int
465vtpci_with_feature(device_t dev, uint64_t feature)
466{
467 struct vtpci_softc *sc;
468
469 sc = device_get_softc(dev);
470
471 return ((sc->vtpci_features & feature) != 0);
472}
473
474static int
475vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
476 struct vq_alloc_info *vq_info)
477{
478 struct vtpci_softc *sc;
479 struct virtqueue *vq;
480 struct vtpci_virtqueue *vqx;
481 struct vq_alloc_info *info;
482 int idx, error;
483 uint16_t size;
484
485 sc = device_get_softc(dev);
486
487 if (sc->vtpci_nvqs != 0)
488 return (EALREADY);
489 if (nvqs <= 0)
490 return (EINVAL);
491
492 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
493 M_DEVBUF, M_NOWAIT | M_ZERO);
494 if (sc->vtpci_vqs == NULL)
495 return (ENOMEM);
496
497 for (idx = 0; idx < nvqs; idx++) {
498 vqx = &sc->vtpci_vqs[idx];
499 info = &vq_info[idx];
500
501 vtpci_select_virtqueue(sc, idx);
502 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
503
504 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
505 0xFFFFFFFFUL, info, &vq);
506 if (error) {
507 device_printf(dev,
508 "cannot allocate virtqueue %d: %d\n", idx, error);
509 break;
510 }
511
512 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
513 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
514
515 vqx->vtv_vq = *info->vqai_vq = vq;
516 vqx->vtv_no_intr = info->vqai_intr == NULL;
517
518 sc->vtpci_nvqs++;
519 }
520
521 if (error)
522 vtpci_free_virtqueues(sc);
523
524 return (error);
525}
526
527static int
528vtpci_setup_intr(device_t dev, enum intr_type type)
529{
530 struct vtpci_softc *sc;
531 int attempt, error;
532
533 sc = device_get_softc(dev);
534
535 for (attempt = 0; attempt < 5; attempt++) {
536 /*
537 * Start with the most desirable interrupt configuration and
538 * fallback towards less desirable ones.
539 */
540 switch (attempt) {
541 case 0:
542 error = vtpci_alloc_intr_msix_pervq(sc);
543 break;
544 case 1:
545 error = vtpci_alloc_intr_msix_shared(sc);
546 break;
547 case 2:
548 error = vtpci_alloc_intr_msi(sc);
549 break;
550 case 3:
551 error = vtpci_alloc_intr_legacy(sc);
552 break;
553 default:
554 device_printf(dev,
555 "exhausted all interrupt allocation attempts\n");
556 return (ENXIO);
557 }
558
559 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
560 break;
561
562 vtpci_cleanup_setup_intr_attempt(sc);
563 }
564
565 if (bootverbose) {
566 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
567 device_printf(dev, "using legacy interrupt\n");
568 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
569 device_printf(dev, "using MSI interrupt\n");
570 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
571 device_printf(dev, "using shared MSIX interrupts\n");
572 else
573 device_printf(dev, "using per VQ MSIX interrupts\n");
574 }
575
576 return (0);
577}
578
579static void
580vtpci_stop(device_t dev)
581{
582
583 vtpci_reset(device_get_softc(dev));
584}
585
586static int
587vtpci_reinit(device_t dev, uint64_t features)
588{
589 struct vtpci_softc *sc;
590 int idx, error;
591
592 sc = device_get_softc(dev);
593
594 /*
595 * Redrive the device initialization. This is a bit of an abuse of
596 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
597 * play nice.
598 *
599 * We do not allow the host device to change from what was originally
600 * negotiated beyond what the guest driver changed. MSIX state should
601 * not change, number of virtqueues and their size remain the same, etc.
602 * This will need to be rethought when we want to support migration.
603 */
604
605 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
606 vtpci_stop(dev);
607
608 /*
609 * Quickly drive the status through ACK and DRIVER. The device
610 * does not become usable again until vtpci_reinit_complete().
611 */
612 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
613 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
614
615 vtpci_negotiate_features(dev, features);
616
617 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
618 error = vtpci_reinit_virtqueue(sc, idx);
619 if (error)
620 return (error);
621 }
622
623 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
624 error = vtpci_set_host_msix_vectors(sc);
625 if (error)
626 return (error);
627 }
628
629 return (0);
630}
631
632static void
633vtpci_reinit_complete(device_t dev)
634{
635
636 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
637}
638
639static void
640vtpci_notify_virtqueue(device_t dev, uint16_t queue)
641{
642 struct vtpci_softc *sc;
643
644 sc = device_get_softc(dev);
645
646 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
647}
648
649static uint8_t
650vtpci_get_status(device_t dev)
651{
652 struct vtpci_softc *sc;
653
654 sc = device_get_softc(dev);
655
656 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
657}
658
659static void
660vtpci_set_status(device_t dev, uint8_t status)
661{
662 struct vtpci_softc *sc;
663
664 sc = device_get_softc(dev);
665
666 if (status != VIRTIO_CONFIG_STATUS_RESET)
667 status |= vtpci_get_status(dev);
668
669 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
670}
671
672static void
673vtpci_read_dev_config(device_t dev, bus_size_t offset,
674 void *dst, int length)
675{
676 struct vtpci_softc *sc;
677 bus_size_t off;
678 uint8_t *d;
679 int size;
680
681 sc = device_get_softc(dev);
682 off = VIRTIO_PCI_CONFIG(sc) + offset;
683
684 for (d = dst; length > 0; d += size, off += size, length -= size) {
685 if (length >= 4) {
686 size = 4;
687 *(uint32_t *)d = vtpci_read_config_4(sc, off);
688 } else if (length >= 2) {
689 size = 2;
690 *(uint16_t *)d = vtpci_read_config_2(sc, off);
691 } else {
692 size = 1;
693 *d = vtpci_read_config_1(sc, off);
694 }
695 }
696}
697
698static void
699vtpci_write_dev_config(device_t dev, bus_size_t offset,
700 void *src, int length)
701{
702 struct vtpci_softc *sc;
703 bus_size_t off;
704 uint8_t *s;
705 int size;
706
707 sc = device_get_softc(dev);
708 off = VIRTIO_PCI_CONFIG(sc) + offset;
709
710 for (s = src; length > 0; s += size, off += size, length -= size) {
711 if (length >= 4) {
712 size = 4;
713 vtpci_write_config_4(sc, off, *(uint32_t *)s);
714 } else if (length >= 2) {
715 size = 2;
716 vtpci_write_config_2(sc, off, *(uint16_t *)s);
717 } else {
718 size = 1;
719 vtpci_write_config_1(sc, off, *s);
720 }
721 }
722}
723
724static void
725vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
726 uint64_t features)
727{
728 device_t dev, child;
729
730 dev = sc->vtpci_dev;
731 child = sc->vtpci_child_dev;
732
733 if (device_is_attached(child) && bootverbose == 0)
734 return;
735
736 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
737}
738
739static void
740vtpci_probe_and_attach_child(struct vtpci_softc *sc)
741{
742 device_t dev, child;
743
744 dev = sc->vtpci_dev;
745 child = sc->vtpci_child_dev;
746
747 if (child == NULL)
748 return;
749
750 if (device_get_state(child) != DS_NOTPRESENT)
751 return;
752
753 if (device_probe(child) != 0)
754 return;
755
756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
757 if (device_attach(child) != 0) {
758 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
759 vtpci_reset(sc);
760 vtpci_release_child_resources(sc);
761 /* Reset status for future attempt. */
762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
763 } else {
764 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
765 VIRTIO_ATTACH_COMPLETED(child);
766 }
767}
768
769static int
770vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
771{
772 device_t dev;
773 int nmsix, cnt, required;
774
775 dev = sc->vtpci_dev;
776
777 /* Allocate an additional vector for the config changes. */
778 required = nvectors + 1;
779
780 nmsix = pci_msix_count(dev);
781 if (nmsix < required)
782 return (1);
783
784 cnt = required;
785 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
786 sc->vtpci_nmsix_resources = required;
787 return (0);
788 }
789
790 pci_release_msi(dev);
791
792 return (1);
793}
794
795static int
796vtpci_alloc_msi(struct vtpci_softc *sc)
797{
798 device_t dev;
799 int nmsi, cnt, required;
800
801 dev = sc->vtpci_dev;
802 required = 1;
803
804 nmsi = pci_msi_count(dev);
805 if (nmsi < required)
806 return (1);
807
808 cnt = required;
809 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
810 return (0);
811
812 pci_release_msi(dev);
813
814 return (1);
815}
816
817static int
818vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
819{
820 int i, nvectors, error;
821
822 if (vtpci_disable_msix != 0 ||
823 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
824 return (ENOTSUP);
825
826 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
827 if (sc->vtpci_vqs[i].vtv_no_intr == 0)
828 nvectors++;
829 }
830
831 error = vtpci_alloc_msix(sc, nvectors);
832 if (error)
833 return (error);
834
835 sc->vtpci_flags |= VTPCI_FLAG_MSIX;
836
837 return (0);
838}
839
840static int
841vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
842{
843 int error;
844
845 if (vtpci_disable_msix != 0 ||
846 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
847 return (ENOTSUP);
848
849 error = vtpci_alloc_msix(sc, 1);
850 if (error)
851 return (error);
852
853 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
854
855 return (0);
856}
857
858static int
859vtpci_alloc_intr_msi(struct vtpci_softc *sc)
860{
861 int error;
862
863 /* Only BHyVe supports MSI. */
864 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
865 return (ENOTSUP);
866
867 error = vtpci_alloc_msi(sc);
868 if (error)
869 return (error);
870
871 sc->vtpci_flags |= VTPCI_FLAG_MSI;
872
873 return (0);
874}
875
876static int
877vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
878{
879
880 sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
881
882 return (0);
883}
884
885static int
886vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
887 struct vtpci_interrupt *intr)
888{
889 struct resource *irq;
890
891 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
892 if (irq == NULL)
893 return (ENXIO);
894
895 intr->vti_irq = irq;
896 intr->vti_rid = rid;
897
898 return (0);
899}
900
901static int
902vtpci_alloc_intr_resources(struct vtpci_softc *sc)
903{
904 struct vtpci_interrupt *intr;
905 int i, rid, flags, nvq_intrs, error;
906
907 rid = 0;
908 flags = RF_ACTIVE;
909
910 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
911 flags |= RF_SHAREABLE;
912 else
913 rid = 1;
914
915 /*
916 * For legacy and MSI interrupts, this single resource handles all
917 * interrupts. For MSIX, this resource is used for the configuration
918 * changed interrupt.
919 */
920 intr = &sc->vtpci_device_interrupt;
921 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
922 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
923 return (error);
924
925 /* Subtract one for the configuration changed interrupt. */
926 nvq_intrs = sc->vtpci_nmsix_resources - 1;
927
928 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
929 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
930 if (sc->vtpci_msix_vq_interrupts == NULL)
931 return (ENOMEM);
932
933 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
934 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
935 if (error)
936 return (error);
937 }
938
939 return (0);
940}
941
942static int
943vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
944{
945 struct vtpci_interrupt *intr;
946 int error;
947
948 intr = &sc->vtpci_device_interrupt;
949 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
950 vtpci_legacy_intr, sc, &intr->vti_handler);
951
952 return (error);
953}
954
955static int
956vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
957{
958 struct vtpci_virtqueue *vqx;
959 struct vtpci_interrupt *intr;
960 int i, error;
961
962 intr = sc->vtpci_msix_vq_interrupts;
963
964 for (i = 0; i < sc->vtpci_nvqs; i++) {
965 vqx = &sc->vtpci_vqs[i];
966
967 if (vqx->vtv_no_intr)
968 continue;
969
970 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
971 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
972 &intr->vti_handler);
973 if (error)
974 return (error);
975
976 intr++;
977 }
978
979 return (0);
980}
981
982static int
983vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
984{
985 device_t dev;
986 struct vtpci_interrupt *intr;
987 int error;
988
989 dev = sc->vtpci_dev;
990 intr = &sc->vtpci_device_interrupt;
991
992 error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
993 vtpci_config_intr, sc, &intr->vti_handler);
994 if (error)
995 return (error);
996
997 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
998 intr = sc->vtpci_msix_vq_interrupts;
999 error = bus_setup_intr(dev, intr->vti_irq, type,
1000 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
1001 &intr->vti_handler);
1002 } else
1003 error = vtpci_setup_pervq_msix_interrupts(sc, type);
1004
1005 return (error ? error : vtpci_set_host_msix_vectors(sc));
1006}
1007
1008static int
1009vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1010{
1011 int error;
1012
1013 type |= INTR_MPSAFE;
1014 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1015 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1016
1017 error = vtpci_alloc_intr_resources(sc);
1018 if (error)
1019 return (error);
1020
1021 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1022 error = vtpci_setup_legacy_interrupt(sc, type);
1023 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1024 error = vtpci_setup_msi_interrupt(sc, type);
1025 else
1026 error = vtpci_setup_msix_interrupts(sc, type);
1027
1028 return (error);
1029}
1030
1031static int
1032vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1033 struct vtpci_interrupt *intr)
1034{
1035 device_t dev;
1036 uint16_t vector;
1037
1038 dev = sc->vtpci_dev;
1039
1040 if (intr != NULL) {
1041 /* Map from guest rid to host vector. */
1042 vector = intr->vti_rid - 1;
1043 } else
1044 vector = VIRTIO_MSI_NO_VECTOR;
1045
1046 vtpci_write_config_2(sc, offset, vector);
1047
1048 /* Read vector to determine if the host had sufficient resources. */
1049 if (vtpci_read_config_2(sc, offset) != vector) {
1050 device_printf(dev,
1051 "insufficient host resources for MSIX interrupts\n");
1052 return (ENODEV);
1053 }
1054
1055 return (0);
1056}
1057
1058static int
1059vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1060{
1061 struct vtpci_interrupt *intr, *tintr;
1062 int idx, offset, error;
1063
1064 intr = &sc->vtpci_device_interrupt;
1065 offset = VIRTIO_MSI_CONFIG_VECTOR;
1066
1067 error = vtpci_register_msix_vector(sc, offset, intr);
1068 if (error)
1069 return (error);
1070
1071 intr = sc->vtpci_msix_vq_interrupts;
1072 offset = VIRTIO_MSI_QUEUE_VECTOR;
1073
1074 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1075 vtpci_select_virtqueue(sc, idx);
1076
1077 if (sc->vtpci_vqs[idx].vtv_no_intr)
1078 tintr = NULL;
1079 else
1080 tintr = intr;
1081
1082 error = vtpci_register_msix_vector(sc, offset, tintr);
1083 if (error)
1084 break;
1085
1086 /*
1087 * For shared MSIX, all the virtqueues share the first
1088 * interrupt.
1089 */
1090 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1091 intr++;
1092 }
1093
1094 return (error);
1095}
1096
1097static int
1098vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1099{
1100 struct vtpci_virtqueue *vqx;
1101 struct virtqueue *vq;
1102 int error;
1103 uint16_t size;
1104
1105 vqx = &sc->vtpci_vqs[idx];
1106 vq = vqx->vtv_vq;
1107
1108 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1109
1110 vtpci_select_virtqueue(sc, idx);
1111 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1112
1113 error = virtqueue_reinit(vq, size);
1114 if (error)
1115 return (error);
1116
1117 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1118 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1119
1120 return (0);
1121}
1122
1123static void
1124vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1125{
1126 device_t dev;
1127
1128 dev = sc->vtpci_dev;
1129
1130 if (intr->vti_handler != NULL) {
1131 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1132 intr->vti_handler = NULL;
1133 }
1134
1135 if (intr->vti_irq != NULL) {
1136 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1137 intr->vti_irq);
1138 intr->vti_irq = NULL;
1139 intr->vti_rid = -1;
1140 }
1141}
1142
1143static void
1144vtpci_free_interrupts(struct vtpci_softc *sc)
1145{
1146 struct vtpci_interrupt *intr;
1147 int i, nvq_intrs;
1148
1149 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1150
1151 if (sc->vtpci_nmsix_resources != 0) {
1152 nvq_intrs = sc->vtpci_nmsix_resources - 1;
1153 sc->vtpci_nmsix_resources = 0;
1154
1155 intr = sc->vtpci_msix_vq_interrupts;
1156 if (intr != NULL) {
1157 for (i = 0; i < nvq_intrs; i++, intr++)
1158 vtpci_free_interrupt(sc, intr);
1159
1160 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1161 sc->vtpci_msix_vq_interrupts = NULL;
1162 }
1163 }
1164
1165 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1166 pci_release_msi(sc->vtpci_dev);
1167
1168 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1169}
1170
1171static void
1172vtpci_free_virtqueues(struct vtpci_softc *sc)
1173{
1174 struct vtpci_virtqueue *vqx;
1175 int idx;
1176
1177 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1178 vqx = &sc->vtpci_vqs[idx];
1179
1180 vtpci_select_virtqueue(sc, idx);
1181 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1182
1183 virtqueue_free(vqx->vtv_vq);
1184 vqx->vtv_vq = NULL;
1185 }
1186
1187 free(sc->vtpci_vqs, M_DEVBUF);
1188 sc->vtpci_vqs = NULL;
1189 sc->vtpci_nvqs = 0;
1190}
1191
1192static void
1193vtpci_release_child_resources(struct vtpci_softc *sc)
1194{
1195
1196 vtpci_free_interrupts(sc);
1197 vtpci_free_virtqueues(sc);
1198}
1199
1200static void
1201vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1202{
1203 int idx;
1204
1205 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1206 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1207 VIRTIO_MSI_NO_VECTOR);
1208
1209 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1210 vtpci_select_virtqueue(sc, idx);
1211 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1212 VIRTIO_MSI_NO_VECTOR);
1213 }
1214 }
1215
1216 vtpci_free_interrupts(sc);
1217}
1218
1219static void
1220vtpci_reset(struct vtpci_softc *sc)
1221{
1222
1223 /*
1224 * Setting the status to RESET sets the host device to
1225 * the original, uninitialized state.
1226 */
1227 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1228}
1229
1230static void
1231vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1232{
1233
1234 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1235}
1236
1237static void
1238vtpci_legacy_intr(void *xsc)
1239{
1240 struct vtpci_softc *sc;
1241 struct vtpci_virtqueue *vqx;
1242 int i;
1243 uint8_t isr;
1244
1245 sc = xsc;
1246 vqx = &sc->vtpci_vqs[0];
1247
1248 /* Reading the ISR also clears it. */
1249 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1250
1251 if (isr & VIRTIO_PCI_ISR_CONFIG)
1252 vtpci_config_intr(sc);
1253
1254 if (isr & VIRTIO_PCI_ISR_INTR) {
1255 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1256 if (vqx->vtv_no_intr == 0)
1257 virtqueue_intr(vqx->vtv_vq);
1258 }
1259 }
1260}
1261
1262static int
1263vtpci_vq_shared_intr_filter(void *xsc)
1264{
1265 struct vtpci_softc *sc;
1266 struct vtpci_virtqueue *vqx;
1267 int i, rc;
1268
1269 rc = 0;
1270 sc = xsc;
1271 vqx = &sc->vtpci_vqs[0];
1272
1273 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1274 if (vqx->vtv_no_intr == 0)
1275 rc |= virtqueue_intr_filter(vqx->vtv_vq);
1276 }
1277
1278 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1279}
1280
1281static void
1282vtpci_vq_shared_intr(void *xsc)
1283{
1284 struct vtpci_softc *sc;
1285 struct vtpci_virtqueue *vqx;
1286 int i;
1287
1288 sc = xsc;
1289 vqx = &sc->vtpci_vqs[0];
1290
1291 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1292 if (vqx->vtv_no_intr == 0)
1293 virtqueue_intr(vqx->vtv_vq);
1294 }
1295}
1296
1297static int
1298vtpci_vq_intr_filter(void *xvq)
1299{
1300 struct virtqueue *vq;
1301 int rc;
1302
1303 vq = xvq;
1304 rc = virtqueue_intr_filter(vq);
1305
1306 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1307}
1308
1309static void
1310vtpci_vq_intr(void *xvq)
1311{
1312 struct virtqueue *vq;
1313
1314 vq = xvq;
1315 virtqueue_intr(vq);
1316}
1317
1318static void
1319vtpci_config_intr(void *xsc)
1320{
1321 struct vtpci_softc *sc;
1322 device_t child;
1323
1324 sc = xsc;
1325 child = sc->vtpci_child_dev;
1326
1327 if (child != NULL)
1328 VIRTIO_CONFIG_CHANGE(child);
1329}
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for the VirtIO PCI interface. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.c 268010 2014-06-29 00:37:59Z bryanv $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/malloc.h>
38
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus.h>
42#include <sys/rman.h>
43
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46
47#include <dev/virtio/virtio.h>
48#include <dev/virtio/virtqueue.h>
49#include <dev/virtio/pci/virtio_pci.h>
50
51#include "virtio_bus_if.h"
52#include "virtio_if.h"
53
54struct vtpci_interrupt {
55 struct resource *vti_irq;
56 int vti_rid;
57 void *vti_handler;
58};
59
60struct vtpci_virtqueue {
61 struct virtqueue *vtv_vq;
62 int vtv_no_intr;
63};
64
65struct vtpci_softc {
66 device_t vtpci_dev;
67 struct resource *vtpci_res;
68 struct resource *vtpci_msix_res;
69 uint64_t vtpci_features;
70 uint32_t vtpci_flags;
71#define VTPCI_FLAG_NO_MSI 0x0001
72#define VTPCI_FLAG_NO_MSIX 0x0002
73#define VTPCI_FLAG_LEGACY 0x1000
74#define VTPCI_FLAG_MSI 0x2000
75#define VTPCI_FLAG_MSIX 0x4000
76#define VTPCI_FLAG_SHARED_MSIX 0x8000
77#define VTPCI_FLAG_ITYPE_MASK 0xF000
78
79 /* This "bus" will only ever have one child. */
80 device_t vtpci_child_dev;
81 struct virtio_feature_desc *vtpci_child_feat_desc;
82
83 int vtpci_nvqs;
84 struct vtpci_virtqueue *vtpci_vqs;
85
86 /*
87 * Ideally, each virtqueue that the driver provides a callback for will
88 * receive its own MSIX vector. If there are not sufficient vectors
89 * available, then attempt to have all the VQs share one vector. For
90 * MSIX, the configuration changed notifications must be on their own
91 * vector.
92 *
93 * If MSIX is not available, we will attempt to have the whole device
94 * share one MSI vector, and then, finally, one legacy interrupt.
95 */
96 struct vtpci_interrupt vtpci_device_interrupt;
97 struct vtpci_interrupt *vtpci_msix_vq_interrupts;
98 int vtpci_nmsix_resources;
99};
100
101static int vtpci_probe(device_t);
102static int vtpci_attach(device_t);
103static int vtpci_detach(device_t);
104static int vtpci_suspend(device_t);
105static int vtpci_resume(device_t);
106static int vtpci_shutdown(device_t);
107static void vtpci_driver_added(device_t, driver_t *);
108static void vtpci_child_detached(device_t, device_t);
109static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
110static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
111
112static uint64_t vtpci_negotiate_features(device_t, uint64_t);
113static int vtpci_with_feature(device_t, uint64_t);
114static int vtpci_alloc_virtqueues(device_t, int, int,
115 struct vq_alloc_info *);
116static int vtpci_setup_intr(device_t, enum intr_type);
117static void vtpci_stop(device_t);
118static int vtpci_reinit(device_t, uint64_t);
119static void vtpci_reinit_complete(device_t);
120static void vtpci_notify_virtqueue(device_t, uint16_t);
121static uint8_t vtpci_get_status(device_t);
122static void vtpci_set_status(device_t, uint8_t);
123static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
124static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
125
126static void vtpci_describe_features(struct vtpci_softc *, const char *,
127 uint64_t);
128static void vtpci_probe_and_attach_child(struct vtpci_softc *);
129
130static int vtpci_alloc_msix(struct vtpci_softc *, int);
131static int vtpci_alloc_msi(struct vtpci_softc *);
132static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
133static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
134static int vtpci_alloc_intr_msi(struct vtpci_softc *);
135static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
136static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
137 struct vtpci_interrupt *);
138static int vtpci_alloc_intr_resources(struct vtpci_softc *);
139
140static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
141 enum intr_type);
142static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
143 enum intr_type);
144static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
145 enum intr_type);
146static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
147
148static int vtpci_register_msix_vector(struct vtpci_softc *, int,
149 struct vtpci_interrupt *);
150static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
151static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
152
153static void vtpci_free_interrupt(struct vtpci_softc *,
154 struct vtpci_interrupt *);
155static void vtpci_free_interrupts(struct vtpci_softc *);
156static void vtpci_free_virtqueues(struct vtpci_softc *);
157static void vtpci_release_child_resources(struct vtpci_softc *);
158static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
159static void vtpci_reset(struct vtpci_softc *);
160
161static void vtpci_select_virtqueue(struct vtpci_softc *, int);
162
163static void vtpci_legacy_intr(void *);
164static int vtpci_vq_shared_intr_filter(void *);
165static void vtpci_vq_shared_intr(void *);
166static int vtpci_vq_intr_filter(void *);
167static void vtpci_vq_intr(void *);
168static void vtpci_config_intr(void *);
169
170#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
171
172#define VIRTIO_PCI_CONFIG(_sc) \
173 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
174
175/*
176 * I/O port read/write wrappers.
177 */
178#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
179#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
180#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
181#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
182#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
183#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
184
185/* Tunables. */
186static int vtpci_disable_msix = 0;
187TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
188
189static device_method_t vtpci_methods[] = {
190 /* Device interface. */
191 DEVMETHOD(device_probe, vtpci_probe),
192 DEVMETHOD(device_attach, vtpci_attach),
193 DEVMETHOD(device_detach, vtpci_detach),
194 DEVMETHOD(device_suspend, vtpci_suspend),
195 DEVMETHOD(device_resume, vtpci_resume),
196 DEVMETHOD(device_shutdown, vtpci_shutdown),
197
198 /* Bus interface. */
199 DEVMETHOD(bus_driver_added, vtpci_driver_added),
200 DEVMETHOD(bus_child_detached, vtpci_child_detached),
201 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
202 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
203
204 /* VirtIO bus interface. */
205 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
206 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
207 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
208 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
209 DEVMETHOD(virtio_bus_stop, vtpci_stop),
210 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
211 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
212 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
213 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
214 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
215
216 DEVMETHOD_END
217};
218
219static driver_t vtpci_driver = {
220 "virtio_pci",
221 vtpci_methods,
222 sizeof(struct vtpci_softc)
223};
224
225devclass_t vtpci_devclass;
226
227DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
228MODULE_VERSION(virtio_pci, 1);
229MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
230MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
231
232static int
233vtpci_probe(device_t dev)
234{
235 char desc[36];
236 const char *name;
237
238 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
239 return (ENXIO);
240
241 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
242 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
243 return (ENXIO);
244
245 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
246 return (ENXIO);
247
248 name = virtio_device_name(pci_get_subdevice(dev));
249 if (name == NULL)
250 name = "Unknown";
251
252 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
253 device_set_desc_copy(dev, desc);
254
255 return (BUS_PROBE_DEFAULT);
256}
257
258static int
259vtpci_attach(device_t dev)
260{
261 struct vtpci_softc *sc;
262 device_t child;
263 int rid;
264
265 sc = device_get_softc(dev);
266 sc->vtpci_dev = dev;
267
268 pci_enable_busmaster(dev);
269
270 rid = PCIR_BAR(0);
271 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
272 RF_ACTIVE);
273 if (sc->vtpci_res == NULL) {
274 device_printf(dev, "cannot map I/O space\n");
275 return (ENXIO);
276 }
277
278 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
279 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
280
281 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
282 rid = PCIR_BAR(1);
283 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
284 SYS_RES_MEMORY, &rid, RF_ACTIVE);
285 }
286
287 if (sc->vtpci_msix_res == NULL)
288 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
289
290 vtpci_reset(sc);
291
292 /* Tell the host we've noticed this device. */
293 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
294
295 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
296 device_printf(dev, "cannot create child device\n");
297 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
298 vtpci_detach(dev);
299 return (ENOMEM);
300 }
301
302 sc->vtpci_child_dev = child;
303 vtpci_probe_and_attach_child(sc);
304
305 return (0);
306}
307
308static int
309vtpci_detach(device_t dev)
310{
311 struct vtpci_softc *sc;
312 device_t child;
313 int error;
314
315 sc = device_get_softc(dev);
316
317 if ((child = sc->vtpci_child_dev) != NULL) {
318 error = device_delete_child(dev, child);
319 if (error)
320 return (error);
321 sc->vtpci_child_dev = NULL;
322 }
323
324 vtpci_reset(sc);
325
326 if (sc->vtpci_msix_res != NULL) {
327 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
328 sc->vtpci_msix_res);
329 sc->vtpci_msix_res = NULL;
330 }
331
332 if (sc->vtpci_res != NULL) {
333 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
334 sc->vtpci_res);
335 sc->vtpci_res = NULL;
336 }
337
338 return (0);
339}
340
341static int
342vtpci_suspend(device_t dev)
343{
344
345 return (bus_generic_suspend(dev));
346}
347
348static int
349vtpci_resume(device_t dev)
350{
351
352 return (bus_generic_resume(dev));
353}
354
355static int
356vtpci_shutdown(device_t dev)
357{
358
359 (void) bus_generic_shutdown(dev);
360 /* Forcibly stop the host device. */
361 vtpci_stop(dev);
362
363 return (0);
364}
365
366static void
367vtpci_driver_added(device_t dev, driver_t *driver)
368{
369 struct vtpci_softc *sc;
370
371 sc = device_get_softc(dev);
372
373 vtpci_probe_and_attach_child(sc);
374}
375
376static void
377vtpci_child_detached(device_t dev, device_t child)
378{
379 struct vtpci_softc *sc;
380
381 sc = device_get_softc(dev);
382
383 vtpci_reset(sc);
384 vtpci_release_child_resources(sc);
385}
386
387static int
388vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
389{
390 struct vtpci_softc *sc;
391
392 sc = device_get_softc(dev);
393
394 if (sc->vtpci_child_dev != child)
395 return (ENOENT);
396
397 switch (index) {
398 case VIRTIO_IVAR_DEVTYPE:
399 case VIRTIO_IVAR_SUBDEVICE:
400 *result = pci_get_subdevice(dev);
401 break;
402 case VIRTIO_IVAR_VENDOR:
403 *result = pci_get_vendor(dev);
404 break;
405 case VIRTIO_IVAR_DEVICE:
406 *result = pci_get_device(dev);
407 break;
408 case VIRTIO_IVAR_SUBVENDOR:
409 *result = pci_get_subdevice(dev);
410 break;
411 default:
412 return (ENOENT);
413 }
414
415 return (0);
416}
417
418static int
419vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
420{
421 struct vtpci_softc *sc;
422
423 sc = device_get_softc(dev);
424
425 if (sc->vtpci_child_dev != child)
426 return (ENOENT);
427
428 switch (index) {
429 case VIRTIO_IVAR_FEATURE_DESC:
430 sc->vtpci_child_feat_desc = (void *) value;
431 break;
432 default:
433 return (ENOENT);
434 }
435
436 return (0);
437}
438
439static uint64_t
440vtpci_negotiate_features(device_t dev, uint64_t child_features)
441{
442 struct vtpci_softc *sc;
443 uint64_t host_features, features;
444
445 sc = device_get_softc(dev);
446
447 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
448 vtpci_describe_features(sc, "host", host_features);
449
450 /*
451 * Limit negotiated features to what the driver, virtqueue, and
452 * host all support.
453 */
454 features = host_features & child_features;
455 features = virtqueue_filter_features(features);
456 sc->vtpci_features = features;
457
458 vtpci_describe_features(sc, "negotiated", features);
459 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
460
461 return (features);
462}
463
464static int
465vtpci_with_feature(device_t dev, uint64_t feature)
466{
467 struct vtpci_softc *sc;
468
469 sc = device_get_softc(dev);
470
471 return ((sc->vtpci_features & feature) != 0);
472}
473
474static int
475vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
476 struct vq_alloc_info *vq_info)
477{
478 struct vtpci_softc *sc;
479 struct virtqueue *vq;
480 struct vtpci_virtqueue *vqx;
481 struct vq_alloc_info *info;
482 int idx, error;
483 uint16_t size;
484
485 sc = device_get_softc(dev);
486
487 if (sc->vtpci_nvqs != 0)
488 return (EALREADY);
489 if (nvqs <= 0)
490 return (EINVAL);
491
492 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
493 M_DEVBUF, M_NOWAIT | M_ZERO);
494 if (sc->vtpci_vqs == NULL)
495 return (ENOMEM);
496
497 for (idx = 0; idx < nvqs; idx++) {
498 vqx = &sc->vtpci_vqs[idx];
499 info = &vq_info[idx];
500
501 vtpci_select_virtqueue(sc, idx);
502 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
503
504 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
505 0xFFFFFFFFUL, info, &vq);
506 if (error) {
507 device_printf(dev,
508 "cannot allocate virtqueue %d: %d\n", idx, error);
509 break;
510 }
511
512 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
513 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
514
515 vqx->vtv_vq = *info->vqai_vq = vq;
516 vqx->vtv_no_intr = info->vqai_intr == NULL;
517
518 sc->vtpci_nvqs++;
519 }
520
521 if (error)
522 vtpci_free_virtqueues(sc);
523
524 return (error);
525}
526
527static int
528vtpci_setup_intr(device_t dev, enum intr_type type)
529{
530 struct vtpci_softc *sc;
531 int attempt, error;
532
533 sc = device_get_softc(dev);
534
535 for (attempt = 0; attempt < 5; attempt++) {
536 /*
537 * Start with the most desirable interrupt configuration and
538 * fallback towards less desirable ones.
539 */
540 switch (attempt) {
541 case 0:
542 error = vtpci_alloc_intr_msix_pervq(sc);
543 break;
544 case 1:
545 error = vtpci_alloc_intr_msix_shared(sc);
546 break;
547 case 2:
548 error = vtpci_alloc_intr_msi(sc);
549 break;
550 case 3:
551 error = vtpci_alloc_intr_legacy(sc);
552 break;
553 default:
554 device_printf(dev,
555 "exhausted all interrupt allocation attempts\n");
556 return (ENXIO);
557 }
558
559 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
560 break;
561
562 vtpci_cleanup_setup_intr_attempt(sc);
563 }
564
565 if (bootverbose) {
566 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
567 device_printf(dev, "using legacy interrupt\n");
568 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
569 device_printf(dev, "using MSI interrupt\n");
570 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
571 device_printf(dev, "using shared MSIX interrupts\n");
572 else
573 device_printf(dev, "using per VQ MSIX interrupts\n");
574 }
575
576 return (0);
577}
578
579static void
580vtpci_stop(device_t dev)
581{
582
583 vtpci_reset(device_get_softc(dev));
584}
585
586static int
587vtpci_reinit(device_t dev, uint64_t features)
588{
589 struct vtpci_softc *sc;
590 int idx, error;
591
592 sc = device_get_softc(dev);
593
594 /*
595 * Redrive the device initialization. This is a bit of an abuse of
596 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
597 * play nice.
598 *
599 * We do not allow the host device to change from what was originally
600 * negotiated beyond what the guest driver changed. MSIX state should
601 * not change, number of virtqueues and their size remain the same, etc.
602 * This will need to be rethought when we want to support migration.
603 */
604
605 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
606 vtpci_stop(dev);
607
608 /*
609 * Quickly drive the status through ACK and DRIVER. The device
610 * does not become usable again until vtpci_reinit_complete().
611 */
612 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
613 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
614
615 vtpci_negotiate_features(dev, features);
616
617 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
618 error = vtpci_reinit_virtqueue(sc, idx);
619 if (error)
620 return (error);
621 }
622
623 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
624 error = vtpci_set_host_msix_vectors(sc);
625 if (error)
626 return (error);
627 }
628
629 return (0);
630}
631
632static void
633vtpci_reinit_complete(device_t dev)
634{
635
636 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
637}
638
639static void
640vtpci_notify_virtqueue(device_t dev, uint16_t queue)
641{
642 struct vtpci_softc *sc;
643
644 sc = device_get_softc(dev);
645
646 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
647}
648
649static uint8_t
650vtpci_get_status(device_t dev)
651{
652 struct vtpci_softc *sc;
653
654 sc = device_get_softc(dev);
655
656 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
657}
658
659static void
660vtpci_set_status(device_t dev, uint8_t status)
661{
662 struct vtpci_softc *sc;
663
664 sc = device_get_softc(dev);
665
666 if (status != VIRTIO_CONFIG_STATUS_RESET)
667 status |= vtpci_get_status(dev);
668
669 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
670}
671
672static void
673vtpci_read_dev_config(device_t dev, bus_size_t offset,
674 void *dst, int length)
675{
676 struct vtpci_softc *sc;
677 bus_size_t off;
678 uint8_t *d;
679 int size;
680
681 sc = device_get_softc(dev);
682 off = VIRTIO_PCI_CONFIG(sc) + offset;
683
684 for (d = dst; length > 0; d += size, off += size, length -= size) {
685 if (length >= 4) {
686 size = 4;
687 *(uint32_t *)d = vtpci_read_config_4(sc, off);
688 } else if (length >= 2) {
689 size = 2;
690 *(uint16_t *)d = vtpci_read_config_2(sc, off);
691 } else {
692 size = 1;
693 *d = vtpci_read_config_1(sc, off);
694 }
695 }
696}
697
698static void
699vtpci_write_dev_config(device_t dev, bus_size_t offset,
700 void *src, int length)
701{
702 struct vtpci_softc *sc;
703 bus_size_t off;
704 uint8_t *s;
705 int size;
706
707 sc = device_get_softc(dev);
708 off = VIRTIO_PCI_CONFIG(sc) + offset;
709
710 for (s = src; length > 0; s += size, off += size, length -= size) {
711 if (length >= 4) {
712 size = 4;
713 vtpci_write_config_4(sc, off, *(uint32_t *)s);
714 } else if (length >= 2) {
715 size = 2;
716 vtpci_write_config_2(sc, off, *(uint16_t *)s);
717 } else {
718 size = 1;
719 vtpci_write_config_1(sc, off, *s);
720 }
721 }
722}
723
724static void
725vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
726 uint64_t features)
727{
728 device_t dev, child;
729
730 dev = sc->vtpci_dev;
731 child = sc->vtpci_child_dev;
732
733 if (device_is_attached(child) && bootverbose == 0)
734 return;
735
736 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
737}
738
739static void
740vtpci_probe_and_attach_child(struct vtpci_softc *sc)
741{
742 device_t dev, child;
743
744 dev = sc->vtpci_dev;
745 child = sc->vtpci_child_dev;
746
747 if (child == NULL)
748 return;
749
750 if (device_get_state(child) != DS_NOTPRESENT)
751 return;
752
753 if (device_probe(child) != 0)
754 return;
755
756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
757 if (device_attach(child) != 0) {
758 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
759 vtpci_reset(sc);
760 vtpci_release_child_resources(sc);
761 /* Reset status for future attempt. */
762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
763 } else {
764 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
765 VIRTIO_ATTACH_COMPLETED(child);
766 }
767}
768
769static int
770vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
771{
772 device_t dev;
773 int nmsix, cnt, required;
774
775 dev = sc->vtpci_dev;
776
777 /* Allocate an additional vector for the config changes. */
778 required = nvectors + 1;
779
780 nmsix = pci_msix_count(dev);
781 if (nmsix < required)
782 return (1);
783
784 cnt = required;
785 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
786 sc->vtpci_nmsix_resources = required;
787 return (0);
788 }
789
790 pci_release_msi(dev);
791
792 return (1);
793}
794
795static int
796vtpci_alloc_msi(struct vtpci_softc *sc)
797{
798 device_t dev;
799 int nmsi, cnt, required;
800
801 dev = sc->vtpci_dev;
802 required = 1;
803
804 nmsi = pci_msi_count(dev);
805 if (nmsi < required)
806 return (1);
807
808 cnt = required;
809 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
810 return (0);
811
812 pci_release_msi(dev);
813
814 return (1);
815}
816
817static int
818vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
819{
820 int i, nvectors, error;
821
822 if (vtpci_disable_msix != 0 ||
823 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
824 return (ENOTSUP);
825
826 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
827 if (sc->vtpci_vqs[i].vtv_no_intr == 0)
828 nvectors++;
829 }
830
831 error = vtpci_alloc_msix(sc, nvectors);
832 if (error)
833 return (error);
834
835 sc->vtpci_flags |= VTPCI_FLAG_MSIX;
836
837 return (0);
838}
839
840static int
841vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
842{
843 int error;
844
845 if (vtpci_disable_msix != 0 ||
846 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
847 return (ENOTSUP);
848
849 error = vtpci_alloc_msix(sc, 1);
850 if (error)
851 return (error);
852
853 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
854
855 return (0);
856}
857
858static int
859vtpci_alloc_intr_msi(struct vtpci_softc *sc)
860{
861 int error;
862
863 /* Only BHyVe supports MSI. */
864 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
865 return (ENOTSUP);
866
867 error = vtpci_alloc_msi(sc);
868 if (error)
869 return (error);
870
871 sc->vtpci_flags |= VTPCI_FLAG_MSI;
872
873 return (0);
874}
875
876static int
877vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
878{
879
880 sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
881
882 return (0);
883}
884
885static int
886vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
887 struct vtpci_interrupt *intr)
888{
889 struct resource *irq;
890
891 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
892 if (irq == NULL)
893 return (ENXIO);
894
895 intr->vti_irq = irq;
896 intr->vti_rid = rid;
897
898 return (0);
899}
900
901static int
902vtpci_alloc_intr_resources(struct vtpci_softc *sc)
903{
904 struct vtpci_interrupt *intr;
905 int i, rid, flags, nvq_intrs, error;
906
907 rid = 0;
908 flags = RF_ACTIVE;
909
910 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
911 flags |= RF_SHAREABLE;
912 else
913 rid = 1;
914
915 /*
916 * For legacy and MSI interrupts, this single resource handles all
917 * interrupts. For MSIX, this resource is used for the configuration
918 * changed interrupt.
919 */
920 intr = &sc->vtpci_device_interrupt;
921 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
922 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
923 return (error);
924
925 /* Subtract one for the configuration changed interrupt. */
926 nvq_intrs = sc->vtpci_nmsix_resources - 1;
927
928 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
929 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
930 if (sc->vtpci_msix_vq_interrupts == NULL)
931 return (ENOMEM);
932
933 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
934 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
935 if (error)
936 return (error);
937 }
938
939 return (0);
940}
941
942static int
943vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
944{
945 struct vtpci_interrupt *intr;
946 int error;
947
948 intr = &sc->vtpci_device_interrupt;
949 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
950 vtpci_legacy_intr, sc, &intr->vti_handler);
951
952 return (error);
953}
954
955static int
956vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
957{
958 struct vtpci_virtqueue *vqx;
959 struct vtpci_interrupt *intr;
960 int i, error;
961
962 intr = sc->vtpci_msix_vq_interrupts;
963
964 for (i = 0; i < sc->vtpci_nvqs; i++) {
965 vqx = &sc->vtpci_vqs[i];
966
967 if (vqx->vtv_no_intr)
968 continue;
969
970 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
971 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
972 &intr->vti_handler);
973 if (error)
974 return (error);
975
976 intr++;
977 }
978
979 return (0);
980}
981
982static int
983vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
984{
985 device_t dev;
986 struct vtpci_interrupt *intr;
987 int error;
988
989 dev = sc->vtpci_dev;
990 intr = &sc->vtpci_device_interrupt;
991
992 error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
993 vtpci_config_intr, sc, &intr->vti_handler);
994 if (error)
995 return (error);
996
997 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
998 intr = sc->vtpci_msix_vq_interrupts;
999 error = bus_setup_intr(dev, intr->vti_irq, type,
1000 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
1001 &intr->vti_handler);
1002 } else
1003 error = vtpci_setup_pervq_msix_interrupts(sc, type);
1004
1005 return (error ? error : vtpci_set_host_msix_vectors(sc));
1006}
1007
1008static int
1009vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1010{
1011 int error;
1012
1013 type |= INTR_MPSAFE;
1014 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1015 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1016
1017 error = vtpci_alloc_intr_resources(sc);
1018 if (error)
1019 return (error);
1020
1021 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1022 error = vtpci_setup_legacy_interrupt(sc, type);
1023 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1024 error = vtpci_setup_msi_interrupt(sc, type);
1025 else
1026 error = vtpci_setup_msix_interrupts(sc, type);
1027
1028 return (error);
1029}
1030
1031static int
1032vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1033 struct vtpci_interrupt *intr)
1034{
1035 device_t dev;
1036 uint16_t vector;
1037
1038 dev = sc->vtpci_dev;
1039
1040 if (intr != NULL) {
1041 /* Map from guest rid to host vector. */
1042 vector = intr->vti_rid - 1;
1043 } else
1044 vector = VIRTIO_MSI_NO_VECTOR;
1045
1046 vtpci_write_config_2(sc, offset, vector);
1047
1048 /* Read vector to determine if the host had sufficient resources. */
1049 if (vtpci_read_config_2(sc, offset) != vector) {
1050 device_printf(dev,
1051 "insufficient host resources for MSIX interrupts\n");
1052 return (ENODEV);
1053 }
1054
1055 return (0);
1056}
1057
1058static int
1059vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1060{
1061 struct vtpci_interrupt *intr, *tintr;
1062 int idx, offset, error;
1063
1064 intr = &sc->vtpci_device_interrupt;
1065 offset = VIRTIO_MSI_CONFIG_VECTOR;
1066
1067 error = vtpci_register_msix_vector(sc, offset, intr);
1068 if (error)
1069 return (error);
1070
1071 intr = sc->vtpci_msix_vq_interrupts;
1072 offset = VIRTIO_MSI_QUEUE_VECTOR;
1073
1074 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1075 vtpci_select_virtqueue(sc, idx);
1076
1077 if (sc->vtpci_vqs[idx].vtv_no_intr)
1078 tintr = NULL;
1079 else
1080 tintr = intr;
1081
1082 error = vtpci_register_msix_vector(sc, offset, tintr);
1083 if (error)
1084 break;
1085
1086 /*
1087 * For shared MSIX, all the virtqueues share the first
1088 * interrupt.
1089 */
1090 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1091 intr++;
1092 }
1093
1094 return (error);
1095}
1096
1097static int
1098vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1099{
1100 struct vtpci_virtqueue *vqx;
1101 struct virtqueue *vq;
1102 int error;
1103 uint16_t size;
1104
1105 vqx = &sc->vtpci_vqs[idx];
1106 vq = vqx->vtv_vq;
1107
1108 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1109
1110 vtpci_select_virtqueue(sc, idx);
1111 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1112
1113 error = virtqueue_reinit(vq, size);
1114 if (error)
1115 return (error);
1116
1117 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1118 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1119
1120 return (0);
1121}
1122
1123static void
1124vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1125{
1126 device_t dev;
1127
1128 dev = sc->vtpci_dev;
1129
1130 if (intr->vti_handler != NULL) {
1131 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1132 intr->vti_handler = NULL;
1133 }
1134
1135 if (intr->vti_irq != NULL) {
1136 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1137 intr->vti_irq);
1138 intr->vti_irq = NULL;
1139 intr->vti_rid = -1;
1140 }
1141}
1142
1143static void
1144vtpci_free_interrupts(struct vtpci_softc *sc)
1145{
1146 struct vtpci_interrupt *intr;
1147 int i, nvq_intrs;
1148
1149 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1150
1151 if (sc->vtpci_nmsix_resources != 0) {
1152 nvq_intrs = sc->vtpci_nmsix_resources - 1;
1153 sc->vtpci_nmsix_resources = 0;
1154
1155 intr = sc->vtpci_msix_vq_interrupts;
1156 if (intr != NULL) {
1157 for (i = 0; i < nvq_intrs; i++, intr++)
1158 vtpci_free_interrupt(sc, intr);
1159
1160 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1161 sc->vtpci_msix_vq_interrupts = NULL;
1162 }
1163 }
1164
1165 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1166 pci_release_msi(sc->vtpci_dev);
1167
1168 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1169}
1170
1171static void
1172vtpci_free_virtqueues(struct vtpci_softc *sc)
1173{
1174 struct vtpci_virtqueue *vqx;
1175 int idx;
1176
1177 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1178 vqx = &sc->vtpci_vqs[idx];
1179
1180 vtpci_select_virtqueue(sc, idx);
1181 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1182
1183 virtqueue_free(vqx->vtv_vq);
1184 vqx->vtv_vq = NULL;
1185 }
1186
1187 free(sc->vtpci_vqs, M_DEVBUF);
1188 sc->vtpci_vqs = NULL;
1189 sc->vtpci_nvqs = 0;
1190}
1191
1192static void
1193vtpci_release_child_resources(struct vtpci_softc *sc)
1194{
1195
1196 vtpci_free_interrupts(sc);
1197 vtpci_free_virtqueues(sc);
1198}
1199
1200static void
1201vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1202{
1203 int idx;
1204
1205 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1206 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1207 VIRTIO_MSI_NO_VECTOR);
1208
1209 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1210 vtpci_select_virtqueue(sc, idx);
1211 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1212 VIRTIO_MSI_NO_VECTOR);
1213 }
1214 }
1215
1216 vtpci_free_interrupts(sc);
1217}
1218
1219static void
1220vtpci_reset(struct vtpci_softc *sc)
1221{
1222
1223 /*
1224 * Setting the status to RESET sets the host device to
1225 * the original, uninitialized state.
1226 */
1227 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1228}
1229
1230static void
1231vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1232{
1233
1234 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1235}
1236
1237static void
1238vtpci_legacy_intr(void *xsc)
1239{
1240 struct vtpci_softc *sc;
1241 struct vtpci_virtqueue *vqx;
1242 int i;
1243 uint8_t isr;
1244
1245 sc = xsc;
1246 vqx = &sc->vtpci_vqs[0];
1247
1248 /* Reading the ISR also clears it. */
1249 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1250
1251 if (isr & VIRTIO_PCI_ISR_CONFIG)
1252 vtpci_config_intr(sc);
1253
1254 if (isr & VIRTIO_PCI_ISR_INTR) {
1255 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1256 if (vqx->vtv_no_intr == 0)
1257 virtqueue_intr(vqx->vtv_vq);
1258 }
1259 }
1260}
1261
1262static int
1263vtpci_vq_shared_intr_filter(void *xsc)
1264{
1265 struct vtpci_softc *sc;
1266 struct vtpci_virtqueue *vqx;
1267 int i, rc;
1268
1269 rc = 0;
1270 sc = xsc;
1271 vqx = &sc->vtpci_vqs[0];
1272
1273 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1274 if (vqx->vtv_no_intr == 0)
1275 rc |= virtqueue_intr_filter(vqx->vtv_vq);
1276 }
1277
1278 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1279}
1280
1281static void
1282vtpci_vq_shared_intr(void *xsc)
1283{
1284 struct vtpci_softc *sc;
1285 struct vtpci_virtqueue *vqx;
1286 int i;
1287
1288 sc = xsc;
1289 vqx = &sc->vtpci_vqs[0];
1290
1291 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1292 if (vqx->vtv_no_intr == 0)
1293 virtqueue_intr(vqx->vtv_vq);
1294 }
1295}
1296
1297static int
1298vtpci_vq_intr_filter(void *xvq)
1299{
1300 struct virtqueue *vq;
1301 int rc;
1302
1303 vq = xvq;
1304 rc = virtqueue_intr_filter(vq);
1305
1306 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1307}
1308
1309static void
1310vtpci_vq_intr(void *xvq)
1311{
1312 struct virtqueue *vq;
1313
1314 vq = xvq;
1315 virtqueue_intr(vq);
1316}
1317
1318static void
1319vtpci_config_intr(void *xsc)
1320{
1321 struct vtpci_softc *sc;
1322 device_t child;
1323
1324 sc = xsc;
1325 child = sc->vtpci_child_dev;
1326
1327 if (child != NULL)
1328 VIRTIO_CONFIG_CHANGE(child);
1329}