1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for the VirtIO PCI interface. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.c 267312 2014-06-10 03:23:35Z bryanv $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/malloc.h>
38
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus.h>
42#include <sys/rman.h>
43
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46
47#include <dev/virtio/virtio.h>
48#include <dev/virtio/virtio_config.h>
49#include <dev/virtio/virtqueue.h>
50#include <dev/virtio/pci/virtio_pci.h>
51
52#include "virtio_bus_if.h"
53#include "virtio_if.h"
54
55struct vtpci_interrupt {
56 struct resource *vti_irq;
57 int vti_rid;
58 void *vti_handler;
59};
60
61struct vtpci_virtqueue {
62 struct virtqueue *vtv_vq;
63 int vtv_no_intr;
64};
65
66struct vtpci_softc {
67 device_t vtpci_dev;
68 struct resource *vtpci_res;
69 struct resource *vtpci_msix_res;
70 uint64_t vtpci_features;
71 uint32_t vtpci_flags;
72#define VTPCI_FLAG_NO_MSI 0x0001
73#define VTPCI_FLAG_NO_MSIX 0x0002
74#define VTPCI_FLAG_LEGACY 0x1000
75#define VTPCI_FLAG_MSI 0x2000
76#define VTPCI_FLAG_MSIX 0x4000
77#define VTPCI_FLAG_SHARED_MSIX 0x8000
78#define VTPCI_FLAG_ITYPE_MASK 0xF000
79
80 /* This "bus" will only ever have one child. */
81 device_t vtpci_child_dev;
82 struct virtio_feature_desc *vtpci_child_feat_desc;
83
84 int vtpci_nvqs;
85 struct vtpci_virtqueue *vtpci_vqs;
86
87 /*
88 * Ideally, each virtqueue that the driver provides a callback for will
89 * receive its own MSIX vector. If there are not sufficient vectors
90 * available, then attempt to have all the VQs share one vector. For
91 * MSIX, the configuration changed notifications must be on their own
92 * vector.
93 *
94 * If MSIX is not available, we will attempt to have the whole device
95 * share one MSI vector, and then, finally, one legacy interrupt.
96 */
97 struct vtpci_interrupt vtpci_device_interrupt;
98 struct vtpci_interrupt *vtpci_msix_vq_interrupts;
99 int vtpci_nmsix_resources;
100};
101
102static int vtpci_probe(device_t);
103static int vtpci_attach(device_t);
104static int vtpci_detach(device_t);
105static int vtpci_suspend(device_t);
106static int vtpci_resume(device_t);
107static int vtpci_shutdown(device_t);
108static void vtpci_driver_added(device_t, driver_t *);
109static void vtpci_child_detached(device_t, device_t);
110static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
111static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
112
113static uint64_t vtpci_negotiate_features(device_t, uint64_t);
114static int vtpci_with_feature(device_t, uint64_t);
115static int vtpci_alloc_virtqueues(device_t, int, int,
116 struct vq_alloc_info *);
117static int vtpci_setup_intr(device_t, enum intr_type);
118static void vtpci_stop(device_t);
119static int vtpci_reinit(device_t, uint64_t);
120static void vtpci_reinit_complete(device_t);
121static void vtpci_notify_virtqueue(device_t, uint16_t);
122static uint8_t vtpci_get_status(device_t);
123static void vtpci_set_status(device_t, uint8_t);
124static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
125static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
126
127static void vtpci_describe_features(struct vtpci_softc *, const char *,
128 uint64_t);
129static void vtpci_probe_and_attach_child(struct vtpci_softc *);
130
131static int vtpci_alloc_msix(struct vtpci_softc *, int);
132static int vtpci_alloc_msi(struct vtpci_softc *);
133static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
134static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
135static int vtpci_alloc_intr_msi(struct vtpci_softc *);
136static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
137static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
138 struct vtpci_interrupt *);
139static int vtpci_alloc_intr_resources(struct vtpci_softc *);
140
141static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
142 enum intr_type);
143static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
144 enum intr_type);
145static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
146 enum intr_type);
147static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
148
149static int vtpci_register_msix_vector(struct vtpci_softc *, int,
150 struct vtpci_interrupt *);
151static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
152static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
153
154static void vtpci_free_interrupt(struct vtpci_softc *,
155 struct vtpci_interrupt *);
156static void vtpci_free_interrupts(struct vtpci_softc *);
157static void vtpci_free_virtqueues(struct vtpci_softc *);
158static void vtpci_release_child_resources(struct vtpci_softc *);
159static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
160static void vtpci_reset(struct vtpci_softc *);
161
162static void vtpci_select_virtqueue(struct vtpci_softc *, int);
163
164static void vtpci_legacy_intr(void *);
165static int vtpci_vq_shared_intr_filter(void *);
166static void vtpci_vq_shared_intr(void *);
167static int vtpci_vq_intr_filter(void *);
168static void vtpci_vq_intr(void *);
169static void vtpci_config_intr(void *);
170
171#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
172
173/*
174 * I/O port read/write wrappers.
175 */
176#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
177#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
178#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
179#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
180#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
181#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
182
183/* Tunables. */
184static int vtpci_disable_msix = 0;
185TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
186
187static device_method_t vtpci_methods[] = {
188 /* Device interface. */
189 DEVMETHOD(device_probe, vtpci_probe),
190 DEVMETHOD(device_attach, vtpci_attach),
191 DEVMETHOD(device_detach, vtpci_detach),
192 DEVMETHOD(device_suspend, vtpci_suspend),
193 DEVMETHOD(device_resume, vtpci_resume),
194 DEVMETHOD(device_shutdown, vtpci_shutdown),
195
196 /* Bus interface. */
197 DEVMETHOD(bus_driver_added, vtpci_driver_added),
198 DEVMETHOD(bus_child_detached, vtpci_child_detached),
199 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
200 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
201
202 /* VirtIO bus interface. */
203 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
204 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
205 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
206 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
207 DEVMETHOD(virtio_bus_stop, vtpci_stop),
208 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
209 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
210 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
211 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
212 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
213
214 DEVMETHOD_END
215};
216
217static driver_t vtpci_driver = {
218 "virtio_pci",
219 vtpci_methods,
220 sizeof(struct vtpci_softc)
221};
222
223devclass_t vtpci_devclass;
224
225DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
226MODULE_VERSION(virtio_pci, 1);
227MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
228MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
229
230static int
231vtpci_probe(device_t dev)
232{
233 char desc[36];
234 const char *name;
235
236 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
237 return (ENXIO);
238
239 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
240 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
241 return (ENXIO);
242
243 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
244 return (ENXIO);
245
246 name = virtio_device_name(pci_get_subdevice(dev));
247 if (name == NULL)
248 name = "Unknown";
249
250 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
251 device_set_desc_copy(dev, desc);
252
253 return (BUS_PROBE_DEFAULT);
254}
255
256static int
257vtpci_attach(device_t dev)
258{
259 struct vtpci_softc *sc;
260 device_t child;
261 int rid;
262
263 sc = device_get_softc(dev);
264 sc->vtpci_dev = dev;
265
266 pci_enable_busmaster(dev);
267
268 rid = PCIR_BAR(0);
269 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
270 RF_ACTIVE);
271 if (sc->vtpci_res == NULL) {
272 device_printf(dev, "cannot map I/O space\n");
273 return (ENXIO);
274 }
275
276 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
277 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
278
279 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
280 rid = PCIR_BAR(1);
281 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
282 SYS_RES_MEMORY, &rid, RF_ACTIVE);
283 }
284
285 if (sc->vtpci_msix_res == NULL)
286 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
287
288 vtpci_reset(sc);
289
290 /* Tell the host we've noticed this device. */
291 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
292
293 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
294 device_printf(dev, "cannot create child device\n");
295 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
296 vtpci_detach(dev);
297 return (ENOMEM);
298 }
299
300 sc->vtpci_child_dev = child;
301 vtpci_probe_and_attach_child(sc);
302
303 return (0);
304}
305
306static int
307vtpci_detach(device_t dev)
308{
309 struct vtpci_softc *sc;
310 device_t child;
311 int error;
312
313 sc = device_get_softc(dev);
314
315 if ((child = sc->vtpci_child_dev) != NULL) {
316 error = device_delete_child(dev, child);
317 if (error)
318 return (error);
319 sc->vtpci_child_dev = NULL;
320 }
321
322 vtpci_reset(sc);
323
324 if (sc->vtpci_msix_res != NULL) {
325 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
326 sc->vtpci_msix_res);
327 sc->vtpci_msix_res = NULL;
328 }
329
330 if (sc->vtpci_res != NULL) {
331 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
332 sc->vtpci_res);
333 sc->vtpci_res = NULL;
334 }
335
336 return (0);
337}
338
339static int
340vtpci_suspend(device_t dev)
341{
342
343 return (bus_generic_suspend(dev));
344}
345
346static int
347vtpci_resume(device_t dev)
348{
349
350 return (bus_generic_resume(dev));
351}
352
353static int
354vtpci_shutdown(device_t dev)
355{
356
357 (void) bus_generic_shutdown(dev);
358 /* Forcibly stop the host device. */
359 vtpci_stop(dev);
360
361 return (0);
362}
363
364static void
365vtpci_driver_added(device_t dev, driver_t *driver)
366{
367 struct vtpci_softc *sc;
368
369 sc = device_get_softc(dev);
370
371 vtpci_probe_and_attach_child(sc);
372}
373
374static void
375vtpci_child_detached(device_t dev, device_t child)
376{
377 struct vtpci_softc *sc;
378
379 sc = device_get_softc(dev);
380
381 vtpci_reset(sc);
382 vtpci_release_child_resources(sc);
383}
384
385static int
386vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
387{
388 struct vtpci_softc *sc;
389
390 sc = device_get_softc(dev);
391
392 if (sc->vtpci_child_dev != child)
393 return (ENOENT);
394
395 switch (index) {
396 case VIRTIO_IVAR_DEVTYPE:
397 case VIRTIO_IVAR_SUBDEVICE:
398 *result = pci_get_subdevice(dev);
399 break;
400 case VIRTIO_IVAR_VENDOR:
401 *result = pci_get_vendor(dev);
402 break;
403 case VIRTIO_IVAR_DEVICE:
404 *result = pci_get_device(dev);
405 break;
406 case VIRTIO_IVAR_SUBVENDOR:
407 *result = pci_get_subdevice(dev);
408 break;
409 default:
410 return (ENOENT);
411 }
412
413 return (0);
414}
415
416static int
417vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
418{
419 struct vtpci_softc *sc;
420
421 sc = device_get_softc(dev);
422
423 if (sc->vtpci_child_dev != child)
424 return (ENOENT);
425
426 switch (index) {
427 case VIRTIO_IVAR_FEATURE_DESC:
428 sc->vtpci_child_feat_desc = (void *) value;
429 break;
430 default:
431 return (ENOENT);
432 }
433
434 return (0);
435}
436
437static uint64_t
438vtpci_negotiate_features(device_t dev, uint64_t child_features)
439{
440 struct vtpci_softc *sc;
441 uint64_t host_features, features;
442
443 sc = device_get_softc(dev);
444
445 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
446 vtpci_describe_features(sc, "host", host_features);
447
448 /*
449 * Limit negotiated features to what the driver, virtqueue, and
450 * host all support.
451 */
452 features = host_features & child_features;
453 features = virtqueue_filter_features(features);
454 sc->vtpci_features = features;
455
456 vtpci_describe_features(sc, "negotiated", features);
457 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
458
459 return (features);
460}
461
462static int
463vtpci_with_feature(device_t dev, uint64_t feature)
464{
465 struct vtpci_softc *sc;
466
467 sc = device_get_softc(dev);
468
469 return ((sc->vtpci_features & feature) != 0);
470}
471
472static int
473vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
474 struct vq_alloc_info *vq_info)
475{
476 struct vtpci_softc *sc;
477 struct virtqueue *vq;
478 struct vtpci_virtqueue *vqx;
479 struct vq_alloc_info *info;
480 int idx, error;
481 uint16_t size;
482
483 sc = device_get_softc(dev);
484
485 if (sc->vtpci_nvqs != 0)
486 return (EALREADY);
487 if (nvqs <= 0)
488 return (EINVAL);
489
490 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
491 M_DEVBUF, M_NOWAIT | M_ZERO);
492 if (sc->vtpci_vqs == NULL)
493 return (ENOMEM);
494
495 for (idx = 0; idx < nvqs; idx++) {
496 vqx = &sc->vtpci_vqs[idx];
497 info = &vq_info[idx];
498
499 vtpci_select_virtqueue(sc, idx);
500 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
501
502 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
503 0xFFFFFFFFUL, info, &vq);
504 if (error) {
505 device_printf(dev,
506 "cannot allocate virtqueue %d: %d\n", idx, error);
507 break;
508 }
509
510 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
511 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
512
513 vqx->vtv_vq = *info->vqai_vq = vq;
514 vqx->vtv_no_intr = info->vqai_intr == NULL;
515
516 sc->vtpci_nvqs++;
517 }
518
519 if (error)
520 vtpci_free_virtqueues(sc);
521
522 return (error);
523}
524
525static int
526vtpci_setup_intr(device_t dev, enum intr_type type)
527{
528 struct vtpci_softc *sc;
529 int attempt, error;
530
531 sc = device_get_softc(dev);
532
533 for (attempt = 0; attempt < 5; attempt++) {
534 /*
535 * Start with the most desirable interrupt configuration and
536 * fallback towards less desirable ones.
537 */
538 switch (attempt) {
539 case 0:
540 error = vtpci_alloc_intr_msix_pervq(sc);
541 break;
542 case 1:
543 error = vtpci_alloc_intr_msix_shared(sc);
544 break;
545 case 2:
546 error = vtpci_alloc_intr_msi(sc);
547 break;
548 case 3:
549 error = vtpci_alloc_intr_legacy(sc);
550 break;
551 default:
552 device_printf(dev,
553 "exhausted all interrupt allocation attempts\n");
554 return (ENXIO);
555 }
556
557 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
558 break;
559
560 vtpci_cleanup_setup_intr_attempt(sc);
561 }
562
563 if (bootverbose) {
564 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
565 device_printf(dev, "using legacy interrupt\n");
566 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
567 device_printf(dev, "using MSI interrupt\n");
568 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
569 device_printf(dev, "using shared MSIX interrupts\n");
570 else
571 device_printf(dev, "using per VQ MSIX interrupts\n");
572 }
573
574 return (0);
575}
576
577static void
578vtpci_stop(device_t dev)
579{
580
581 vtpci_reset(device_get_softc(dev));
582}
583
584static int
585vtpci_reinit(device_t dev, uint64_t features)
586{
587 struct vtpci_softc *sc;
588 int idx, error;
589
590 sc = device_get_softc(dev);
591
592 /*
593 * Redrive the device initialization. This is a bit of an abuse of
594 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
595 * play nice.
596 *
597 * We do not allow the host device to change from what was originally
598 * negotiated beyond what the guest driver changed. MSIX state should
599 * not change, number of virtqueues and their size remain the same, etc.
600 * This will need to be rethought when we want to support migration.
601 */
602
603 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
604 vtpci_stop(dev);
605
606 /*
607 * Quickly drive the status through ACK and DRIVER. The device
608 * does not become usable again until vtpci_reinit_complete().
609 */
610 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
611 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
612
613 vtpci_negotiate_features(dev, features);
614
615 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
616 error = vtpci_reinit_virtqueue(sc, idx);
617 if (error)
618 return (error);
619 }
620
621 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
622 error = vtpci_set_host_msix_vectors(sc);
623 if (error)
624 return (error);
625 }
626
627 return (0);
628}
629
630static void
631vtpci_reinit_complete(device_t dev)
632{
633
634 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
635}
636
637static void
638vtpci_notify_virtqueue(device_t dev, uint16_t queue)
639{
640 struct vtpci_softc *sc;
641
642 sc = device_get_softc(dev);
643
644 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
645}
646
647static uint8_t
648vtpci_get_status(device_t dev)
649{
650 struct vtpci_softc *sc;
651
652 sc = device_get_softc(dev);
653
654 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
655}
656
657static void
658vtpci_set_status(device_t dev, uint8_t status)
659{
660 struct vtpci_softc *sc;
661
662 sc = device_get_softc(dev);
663
664 if (status != VIRTIO_CONFIG_STATUS_RESET)
665 status |= vtpci_get_status(dev);
666
667 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
668}
669
670static void
671vtpci_read_dev_config(device_t dev, bus_size_t offset,
672 void *dst, int length)
673{
674 struct vtpci_softc *sc;
675 bus_size_t off;
676 uint8_t *d;
677 int size;
678
679 sc = device_get_softc(dev);
680 off = VIRTIO_PCI_CONFIG(sc) + offset;
681
682 for (d = dst; length > 0; d += size, off += size, length -= size) {
683 if (length >= 4) {
684 size = 4;
685 *(uint32_t *)d = vtpci_read_config_4(sc, off);
686 } else if (length >= 2) {
687 size = 2;
688 *(uint16_t *)d = vtpci_read_config_2(sc, off);
689 } else {
690 size = 1;
691 *d = vtpci_read_config_1(sc, off);
692 }
693 }
694}
695
696static void
697vtpci_write_dev_config(device_t dev, bus_size_t offset,
698 void *src, int length)
699{
700 struct vtpci_softc *sc;
701 bus_size_t off;
702 uint8_t *s;
703 int size;
704
705 sc = device_get_softc(dev);
706 off = VIRTIO_PCI_CONFIG(sc) + offset;
707
708 for (s = src; length > 0; s += size, off += size, length -= size) {
709 if (length >= 4) {
710 size = 4;
711 vtpci_write_config_4(sc, off, *(uint32_t *)s);
712 } else if (length >= 2) {
713 size = 2;
714 vtpci_write_config_2(sc, off, *(uint16_t *)s);
715 } else {
716 size = 1;
717 vtpci_write_config_1(sc, off, *s);
718 }
719 }
720}
721
722static void
723vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
724 uint64_t features)
725{
726 device_t dev, child;
727
728 dev = sc->vtpci_dev;
729 child = sc->vtpci_child_dev;
730
731 if (device_is_attached(child) && bootverbose == 0)
732 return;
733
734 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
735}
736
737static void
738vtpci_probe_and_attach_child(struct vtpci_softc *sc)
739{
740 device_t dev, child;
741
742 dev = sc->vtpci_dev;
743 child = sc->vtpci_child_dev;
744
745 if (child == NULL)
746 return;
747
748 if (device_get_state(child) != DS_NOTPRESENT)
749 return;
750
751 if (device_probe(child) != 0)
752 return;
753
754 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
755 if (device_attach(child) != 0) {
756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
757 vtpci_reset(sc);
758 vtpci_release_child_resources(sc);
759 /* Reset status for future attempt. */
760 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
761 } else {
762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
763 VIRTIO_ATTACH_COMPLETED(child);
764 }
765}
766
767static int
768vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
769{
770 device_t dev;
771 int nmsix, cnt, required;
772
773 dev = sc->vtpci_dev;
774
775 /* Allocate an additional vector for the config changes. */
776 required = nvectors + 1;
777
778 nmsix = pci_msix_count(dev);
779 if (nmsix < required)
780 return (1);
781
782 cnt = required;
783 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
784 sc->vtpci_nmsix_resources = required;
785 return (0);
786 }
787
788 pci_release_msi(dev);
789
790 return (1);
791}
792
793static int
794vtpci_alloc_msi(struct vtpci_softc *sc)
795{
796 device_t dev;
797 int nmsi, cnt, required;
798
799 dev = sc->vtpci_dev;
800 required = 1;
801
802 nmsi = pci_msi_count(dev);
803 if (nmsi < required)
804 return (1);
805
806 cnt = required;
807 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
808 return (0);
809
810 pci_release_msi(dev);
811
812 return (1);
813}
814
815static int
816vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
817{
818 int i, nvectors, error;
819
820 if (vtpci_disable_msix != 0 ||
821 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
822 return (ENOTSUP);
823
824 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
825 if (sc->vtpci_vqs[i].vtv_no_intr == 0)
826 nvectors++;
827 }
828
829 error = vtpci_alloc_msix(sc, nvectors);
830 if (error)
831 return (error);
832
833 sc->vtpci_flags |= VTPCI_FLAG_MSIX;
834
835 return (0);
836}
837
838static int
839vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
840{
841 int error;
842
843 if (vtpci_disable_msix != 0 ||
844 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
845 return (ENOTSUP);
846
847 error = vtpci_alloc_msix(sc, 1);
848 if (error)
849 return (error);
850
851 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
852
853 return (0);
854}
855
856static int
857vtpci_alloc_intr_msi(struct vtpci_softc *sc)
858{
859 int error;
860
861 /* Only BHyVe supports MSI. */
862 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
863 return (ENOTSUP);
864
865 error = vtpci_alloc_msi(sc);
866 if (error)
867 return (error);
868
869 sc->vtpci_flags |= VTPCI_FLAG_MSI;
870
871 return (0);
872}
873
874static int
875vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
876{
877
878 sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
879
880 return (0);
881}
882
883static int
884vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
885 struct vtpci_interrupt *intr)
886{
887 struct resource *irq;
888
889 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
890 if (irq == NULL)
891 return (ENXIO);
892
893 intr->vti_irq = irq;
894 intr->vti_rid = rid;
895
896 return (0);
897}
898
899static int
900vtpci_alloc_intr_resources(struct vtpci_softc *sc)
901{
902 struct vtpci_interrupt *intr;
903 int i, rid, flags, nvq_intrs, error;
904
905 rid = 0;
906 flags = RF_ACTIVE;
907
908 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
909 flags |= RF_SHAREABLE;
910 else
911 rid = 1;
912
913 /*
914 * For legacy and MSI interrupts, this single resource handles all
915 * interrupts. For MSIX, this resource is used for the configuration
916 * changed interrupt.
917 */
918 intr = &sc->vtpci_device_interrupt;
919 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
920 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
921 return (error);
922
923 /* Subtract one for the configuration changed interrupt. */
924 nvq_intrs = sc->vtpci_nmsix_resources - 1;
925
926 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
927 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
928 if (sc->vtpci_msix_vq_interrupts == NULL)
929 return (ENOMEM);
930
931 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
932 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
933 if (error)
934 return (error);
935 }
936
937 return (0);
938}
939
940static int
941vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
942{
943 struct vtpci_interrupt *intr;
944 int error;
945
946 intr = &sc->vtpci_device_interrupt;
947 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
948 vtpci_legacy_intr, sc, &intr->vti_handler);
949
950 return (error);
951}
952
953static int
954vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
955{
956 struct vtpci_virtqueue *vqx;
957 struct vtpci_interrupt *intr;
958 int i, error;
959
960 intr = sc->vtpci_msix_vq_interrupts;
961
962 for (i = 0; i < sc->vtpci_nvqs; i++) {
963 vqx = &sc->vtpci_vqs[i];
964
965 if (vqx->vtv_no_intr)
966 continue;
967
968 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
969 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
970 &intr->vti_handler);
971 if (error)
972 return (error);
973
974 intr++;
975 }
976
977 return (0);
978}
979
980static int
981vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
982{
983 device_t dev;
984 struct vtpci_interrupt *intr;
985 int error;
986
987 dev = sc->vtpci_dev;
988 intr = &sc->vtpci_device_interrupt;
989
990 error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
991 vtpci_config_intr, sc, &intr->vti_handler);
992 if (error)
993 return (error);
994
995 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
996 intr = sc->vtpci_msix_vq_interrupts;
997 error = bus_setup_intr(dev, intr->vti_irq, type,
998 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
999 &intr->vti_handler);
1000 } else
1001 error = vtpci_setup_pervq_msix_interrupts(sc, type);
1002
1003 return (error ? error : vtpci_set_host_msix_vectors(sc));
1004}
1005
1006static int
1007vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1008{
1009 int error;
1010
1011 type |= INTR_MPSAFE;
1012 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1013 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1014
1015 error = vtpci_alloc_intr_resources(sc);
1016 if (error)
1017 return (error);
1018
1019 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1020 error = vtpci_setup_legacy_interrupt(sc, type);
1021 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1022 error = vtpci_setup_msi_interrupt(sc, type);
1023 else
1024 error = vtpci_setup_msix_interrupts(sc, type);
1025
1026 return (error);
1027}
1028
1029static int
1030vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1031 struct vtpci_interrupt *intr)
1032{
1033 device_t dev;
1034 uint16_t vector;
1035
1036 dev = sc->vtpci_dev;
1037
1038 if (intr != NULL) {
1039 /* Map from guest rid to host vector. */
1040 vector = intr->vti_rid - 1;
1041 } else
1042 vector = VIRTIO_MSI_NO_VECTOR;
1043
1044 vtpci_write_config_2(sc, offset, vector);
1045
1046 /* Read vector to determine if the host had sufficient resources. */
1047 if (vtpci_read_config_2(sc, offset) != vector) {
1048 device_printf(dev,
1049 "insufficient host resources for MSIX interrupts\n");
1050 return (ENODEV);
1051 }
1052
1053 return (0);
1054}
1055
1056static int
1057vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1058{
1059 struct vtpci_interrupt *intr, *tintr;
1060 int idx, offset, error;
1061
1062 intr = &sc->vtpci_device_interrupt;
1063 offset = VIRTIO_MSI_CONFIG_VECTOR;
1064
1065 error = vtpci_register_msix_vector(sc, offset, intr);
1066 if (error)
1067 return (error);
1068
1069 intr = sc->vtpci_msix_vq_interrupts;
1070 offset = VIRTIO_MSI_QUEUE_VECTOR;
1071
1072 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1073 vtpci_select_virtqueue(sc, idx);
1074
1075 if (sc->vtpci_vqs[idx].vtv_no_intr)
1076 tintr = NULL;
1077 else
1078 tintr = intr;
1079
1080 error = vtpci_register_msix_vector(sc, offset, tintr);
1081 if (error)
1082 break;
1083
1084 /*
1085 * For shared MSIX, all the virtqueues share the first
1086 * interrupt.
1087 */
1088 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1089 intr++;
1090 }
1091
1092 return (error);
1093}
1094
1095static int
1096vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1097{
1098 struct vtpci_virtqueue *vqx;
1099 struct virtqueue *vq;
1100 int error;
1101 uint16_t size;
1102
1103 vqx = &sc->vtpci_vqs[idx];
1104 vq = vqx->vtv_vq;
1105
1106 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1107
1108 vtpci_select_virtqueue(sc, idx);
1109 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1110
1111 error = virtqueue_reinit(vq, size);
1112 if (error)
1113 return (error);
1114
1115 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1116 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1117
1118 return (0);
1119}
1120
1121static void
1122vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1123{
1124 device_t dev;
1125
1126 dev = sc->vtpci_dev;
1127
1128 if (intr->vti_handler != NULL) {
1129 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1130 intr->vti_handler = NULL;
1131 }
1132
1133 if (intr->vti_irq != NULL) {
1134 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1135 intr->vti_irq);
1136 intr->vti_irq = NULL;
1137 intr->vti_rid = -1;
1138 }
1139}
1140
1141static void
1142vtpci_free_interrupts(struct vtpci_softc *sc)
1143{
1144 struct vtpci_interrupt *intr;
1145 int i, nvq_intrs;
1146
1147 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1148
1149 if (sc->vtpci_nmsix_resources != 0) {
1150 nvq_intrs = sc->vtpci_nmsix_resources - 1;
1151 sc->vtpci_nmsix_resources = 0;
1152
1153 intr = sc->vtpci_msix_vq_interrupts;
1154 if (intr != NULL) {
1155 for (i = 0; i < nvq_intrs; i++, intr++)
1156 vtpci_free_interrupt(sc, intr);
1157
1158 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1159 sc->vtpci_msix_vq_interrupts = NULL;
1160 }
1161 }
1162
1163 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1164 pci_release_msi(sc->vtpci_dev);
1165
1166 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1167}
1168
1169static void
1170vtpci_free_virtqueues(struct vtpci_softc *sc)
1171{
1172 struct vtpci_virtqueue *vqx;
1173 int idx;
1174
1175 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1176 vqx = &sc->vtpci_vqs[idx];
1177
1178 vtpci_select_virtqueue(sc, idx);
1179 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1180
1181 virtqueue_free(vqx->vtv_vq);
1182 vqx->vtv_vq = NULL;
1183 }
1184
1185 free(sc->vtpci_vqs, M_DEVBUF);
1186 sc->vtpci_vqs = NULL;
1187 sc->vtpci_nvqs = 0;
1188}
1189
1190static void
1191vtpci_release_child_resources(struct vtpci_softc *sc)
1192{
1193
1194 vtpci_free_interrupts(sc);
1195 vtpci_free_virtqueues(sc);
1196}
1197
1198static void
1199vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1200{
1201 int idx;
1202
1203 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1204 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1205 VIRTIO_MSI_NO_VECTOR);
1206
1207 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1208 vtpci_select_virtqueue(sc, idx);
1209 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1210 VIRTIO_MSI_NO_VECTOR);
1211 }
1212 }
1213
1214 vtpci_free_interrupts(sc);
1215}
1216
1217static void
1218vtpci_reset(struct vtpci_softc *sc)
1219{
1220
1221 /*
1222 * Setting the status to RESET sets the host device to
1223 * the original, uninitialized state.
1224 */
1225 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1226}
1227
1228static void
1229vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1230{
1231
1232 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1233}
1234
1235static void
1236vtpci_legacy_intr(void *xsc)
1237{
1238 struct vtpci_softc *sc;
1239 struct vtpci_virtqueue *vqx;
1240 int i;
1241 uint8_t isr;
1242
1243 sc = xsc;
1244 vqx = &sc->vtpci_vqs[0];
1245
1246 /* Reading the ISR also clears it. */
1247 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1248
1249 if (isr & VIRTIO_PCI_ISR_CONFIG)
1250 vtpci_config_intr(sc);
1251
1252 if (isr & VIRTIO_PCI_ISR_INTR) {
1253 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1254 if (vqx->vtv_no_intr == 0)
1255 virtqueue_intr(vqx->vtv_vq);
1256 }
1257 }
1258}
1259
1260static int
1261vtpci_vq_shared_intr_filter(void *xsc)
1262{
1263 struct vtpci_softc *sc;
1264 struct vtpci_virtqueue *vqx;
1265 int i, rc;
1266
1267 rc = 0;
1268 sc = xsc;
1269 vqx = &sc->vtpci_vqs[0];
1270
1271 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1272 if (vqx->vtv_no_intr == 0)
1273 rc |= virtqueue_intr_filter(vqx->vtv_vq);
1274 }
1275
1276 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1277}
1278
1279static void
1280vtpci_vq_shared_intr(void *xsc)
1281{
1282 struct vtpci_softc *sc;
1283 struct vtpci_virtqueue *vqx;
1284 int i;
1285
1286 sc = xsc;
1287 vqx = &sc->vtpci_vqs[0];
1288
1289 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1290 if (vqx->vtv_no_intr == 0)
1291 virtqueue_intr(vqx->vtv_vq);
1292 }
1293}
1294
1295static int
1296vtpci_vq_intr_filter(void *xvq)
1297{
1298 struct virtqueue *vq;
1299 int rc;
1300
1301 vq = xvq;
1302 rc = virtqueue_intr_filter(vq);
1303
1304 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1305}
1306
1307static void
1308vtpci_vq_intr(void *xvq)
1309{
1310 struct virtqueue *vq;
1311
1312 vq = xvq;
1313 virtqueue_intr(vq);
1314}
1315
1316static void
1317vtpci_config_intr(void *xsc)
1318{
1319 struct vtpci_softc *sc;
1320 device_t child;
1321
1322 sc = xsc;
1323 child = sc->vtpci_child_dev;
1324
1325 if (child != NULL)
1326 VIRTIO_CONFIG_CHANGE(child);
1327}
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for the VirtIO PCI interface. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/dev/virtio/pci/virtio_pci.c 267312 2014-06-10 03:23:35Z bryanv $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/malloc.h>
38
39#include <machine/bus.h>
40#include <machine/resource.h>
41#include <sys/bus.h>
42#include <sys/rman.h>
43
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46
47#include <dev/virtio/virtio.h>
48#include <dev/virtio/virtio_config.h>
49#include <dev/virtio/virtqueue.h>
50#include <dev/virtio/pci/virtio_pci.h>
51
52#include "virtio_bus_if.h"
53#include "virtio_if.h"
54
55struct vtpci_interrupt {
56 struct resource *vti_irq;
57 int vti_rid;
58 void *vti_handler;
59};
60
61struct vtpci_virtqueue {
62 struct virtqueue *vtv_vq;
63 int vtv_no_intr;
64};
65
66struct vtpci_softc {
67 device_t vtpci_dev;
68 struct resource *vtpci_res;
69 struct resource *vtpci_msix_res;
70 uint64_t vtpci_features;
71 uint32_t vtpci_flags;
72#define VTPCI_FLAG_NO_MSI 0x0001
73#define VTPCI_FLAG_NO_MSIX 0x0002
74#define VTPCI_FLAG_LEGACY 0x1000
75#define VTPCI_FLAG_MSI 0x2000
76#define VTPCI_FLAG_MSIX 0x4000
77#define VTPCI_FLAG_SHARED_MSIX 0x8000
78#define VTPCI_FLAG_ITYPE_MASK 0xF000
79
80 /* This "bus" will only ever have one child. */
81 device_t vtpci_child_dev;
82 struct virtio_feature_desc *vtpci_child_feat_desc;
83
84 int vtpci_nvqs;
85 struct vtpci_virtqueue *vtpci_vqs;
86
87 /*
88 * Ideally, each virtqueue that the driver provides a callback for will
89 * receive its own MSIX vector. If there are not sufficient vectors
90 * available, then attempt to have all the VQs share one vector. For
91 * MSIX, the configuration changed notifications must be on their own
92 * vector.
93 *
94 * If MSIX is not available, we will attempt to have the whole device
95 * share one MSI vector, and then, finally, one legacy interrupt.
96 */
97 struct vtpci_interrupt vtpci_device_interrupt;
98 struct vtpci_interrupt *vtpci_msix_vq_interrupts;
99 int vtpci_nmsix_resources;
100};
101
102static int vtpci_probe(device_t);
103static int vtpci_attach(device_t);
104static int vtpci_detach(device_t);
105static int vtpci_suspend(device_t);
106static int vtpci_resume(device_t);
107static int vtpci_shutdown(device_t);
108static void vtpci_driver_added(device_t, driver_t *);
109static void vtpci_child_detached(device_t, device_t);
110static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
111static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
112
113static uint64_t vtpci_negotiate_features(device_t, uint64_t);
114static int vtpci_with_feature(device_t, uint64_t);
115static int vtpci_alloc_virtqueues(device_t, int, int,
116 struct vq_alloc_info *);
117static int vtpci_setup_intr(device_t, enum intr_type);
118static void vtpci_stop(device_t);
119static int vtpci_reinit(device_t, uint64_t);
120static void vtpci_reinit_complete(device_t);
121static void vtpci_notify_virtqueue(device_t, uint16_t);
122static uint8_t vtpci_get_status(device_t);
123static void vtpci_set_status(device_t, uint8_t);
124static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
125static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
126
127static void vtpci_describe_features(struct vtpci_softc *, const char *,
128 uint64_t);
129static void vtpci_probe_and_attach_child(struct vtpci_softc *);
130
131static int vtpci_alloc_msix(struct vtpci_softc *, int);
132static int vtpci_alloc_msi(struct vtpci_softc *);
133static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
134static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
135static int vtpci_alloc_intr_msi(struct vtpci_softc *);
136static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
137static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
138 struct vtpci_interrupt *);
139static int vtpci_alloc_intr_resources(struct vtpci_softc *);
140
141static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
142 enum intr_type);
143static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
144 enum intr_type);
145static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
146 enum intr_type);
147static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
148
149static int vtpci_register_msix_vector(struct vtpci_softc *, int,
150 struct vtpci_interrupt *);
151static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
152static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
153
154static void vtpci_free_interrupt(struct vtpci_softc *,
155 struct vtpci_interrupt *);
156static void vtpci_free_interrupts(struct vtpci_softc *);
157static void vtpci_free_virtqueues(struct vtpci_softc *);
158static void vtpci_release_child_resources(struct vtpci_softc *);
159static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
160static void vtpci_reset(struct vtpci_softc *);
161
162static void vtpci_select_virtqueue(struct vtpci_softc *, int);
163
164static void vtpci_legacy_intr(void *);
165static int vtpci_vq_shared_intr_filter(void *);
166static void vtpci_vq_shared_intr(void *);
167static int vtpci_vq_intr_filter(void *);
168static void vtpci_vq_intr(void *);
169static void vtpci_config_intr(void *);
170
171#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
172
173/*
174 * I/O port read/write wrappers.
175 */
176#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
177#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
178#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
179#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
180#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
181#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
182
183/* Tunables. */
184static int vtpci_disable_msix = 0;
185TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
186
187static device_method_t vtpci_methods[] = {
188 /* Device interface. */
189 DEVMETHOD(device_probe, vtpci_probe),
190 DEVMETHOD(device_attach, vtpci_attach),
191 DEVMETHOD(device_detach, vtpci_detach),
192 DEVMETHOD(device_suspend, vtpci_suspend),
193 DEVMETHOD(device_resume, vtpci_resume),
194 DEVMETHOD(device_shutdown, vtpci_shutdown),
195
196 /* Bus interface. */
197 DEVMETHOD(bus_driver_added, vtpci_driver_added),
198 DEVMETHOD(bus_child_detached, vtpci_child_detached),
199 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
200 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
201
202 /* VirtIO bus interface. */
203 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
204 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
205 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
206 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
207 DEVMETHOD(virtio_bus_stop, vtpci_stop),
208 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
209 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
210 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
211 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
212 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
213
214 DEVMETHOD_END
215};
216
217static driver_t vtpci_driver = {
218 "virtio_pci",
219 vtpci_methods,
220 sizeof(struct vtpci_softc)
221};
222
223devclass_t vtpci_devclass;
224
225DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
226MODULE_VERSION(virtio_pci, 1);
227MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
228MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
229
230static int
231vtpci_probe(device_t dev)
232{
233 char desc[36];
234 const char *name;
235
236 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
237 return (ENXIO);
238
239 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
240 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
241 return (ENXIO);
242
243 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
244 return (ENXIO);
245
246 name = virtio_device_name(pci_get_subdevice(dev));
247 if (name == NULL)
248 name = "Unknown";
249
250 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
251 device_set_desc_copy(dev, desc);
252
253 return (BUS_PROBE_DEFAULT);
254}
255
256static int
257vtpci_attach(device_t dev)
258{
259 struct vtpci_softc *sc;
260 device_t child;
261 int rid;
262
263 sc = device_get_softc(dev);
264 sc->vtpci_dev = dev;
265
266 pci_enable_busmaster(dev);
267
268 rid = PCIR_BAR(0);
269 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
270 RF_ACTIVE);
271 if (sc->vtpci_res == NULL) {
272 device_printf(dev, "cannot map I/O space\n");
273 return (ENXIO);
274 }
275
276 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
277 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
278
279 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
280 rid = PCIR_BAR(1);
281 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
282 SYS_RES_MEMORY, &rid, RF_ACTIVE);
283 }
284
285 if (sc->vtpci_msix_res == NULL)
286 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
287
288 vtpci_reset(sc);
289
290 /* Tell the host we've noticed this device. */
291 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
292
293 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
294 device_printf(dev, "cannot create child device\n");
295 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
296 vtpci_detach(dev);
297 return (ENOMEM);
298 }
299
300 sc->vtpci_child_dev = child;
301 vtpci_probe_and_attach_child(sc);
302
303 return (0);
304}
305
306static int
307vtpci_detach(device_t dev)
308{
309 struct vtpci_softc *sc;
310 device_t child;
311 int error;
312
313 sc = device_get_softc(dev);
314
315 if ((child = sc->vtpci_child_dev) != NULL) {
316 error = device_delete_child(dev, child);
317 if (error)
318 return (error);
319 sc->vtpci_child_dev = NULL;
320 }
321
322 vtpci_reset(sc);
323
324 if (sc->vtpci_msix_res != NULL) {
325 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
326 sc->vtpci_msix_res);
327 sc->vtpci_msix_res = NULL;
328 }
329
330 if (sc->vtpci_res != NULL) {
331 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
332 sc->vtpci_res);
333 sc->vtpci_res = NULL;
334 }
335
336 return (0);
337}
338
339static int
340vtpci_suspend(device_t dev)
341{
342
343 return (bus_generic_suspend(dev));
344}
345
346static int
347vtpci_resume(device_t dev)
348{
349
350 return (bus_generic_resume(dev));
351}
352
353static int
354vtpci_shutdown(device_t dev)
355{
356
357 (void) bus_generic_shutdown(dev);
358 /* Forcibly stop the host device. */
359 vtpci_stop(dev);
360
361 return (0);
362}
363
364static void
365vtpci_driver_added(device_t dev, driver_t *driver)
366{
367 struct vtpci_softc *sc;
368
369 sc = device_get_softc(dev);
370
371 vtpci_probe_and_attach_child(sc);
372}
373
374static void
375vtpci_child_detached(device_t dev, device_t child)
376{
377 struct vtpci_softc *sc;
378
379 sc = device_get_softc(dev);
380
381 vtpci_reset(sc);
382 vtpci_release_child_resources(sc);
383}
384
385static int
386vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
387{
388 struct vtpci_softc *sc;
389
390 sc = device_get_softc(dev);
391
392 if (sc->vtpci_child_dev != child)
393 return (ENOENT);
394
395 switch (index) {
396 case VIRTIO_IVAR_DEVTYPE:
397 case VIRTIO_IVAR_SUBDEVICE:
398 *result = pci_get_subdevice(dev);
399 break;
400 case VIRTIO_IVAR_VENDOR:
401 *result = pci_get_vendor(dev);
402 break;
403 case VIRTIO_IVAR_DEVICE:
404 *result = pci_get_device(dev);
405 break;
406 case VIRTIO_IVAR_SUBVENDOR:
407 *result = pci_get_subdevice(dev);
408 break;
409 default:
410 return (ENOENT);
411 }
412
413 return (0);
414}
415
416static int
417vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
418{
419 struct vtpci_softc *sc;
420
421 sc = device_get_softc(dev);
422
423 if (sc->vtpci_child_dev != child)
424 return (ENOENT);
425
426 switch (index) {
427 case VIRTIO_IVAR_FEATURE_DESC:
428 sc->vtpci_child_feat_desc = (void *) value;
429 break;
430 default:
431 return (ENOENT);
432 }
433
434 return (0);
435}
436
437static uint64_t
438vtpci_negotiate_features(device_t dev, uint64_t child_features)
439{
440 struct vtpci_softc *sc;
441 uint64_t host_features, features;
442
443 sc = device_get_softc(dev);
444
445 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
446 vtpci_describe_features(sc, "host", host_features);
447
448 /*
449 * Limit negotiated features to what the driver, virtqueue, and
450 * host all support.
451 */
452 features = host_features & child_features;
453 features = virtqueue_filter_features(features);
454 sc->vtpci_features = features;
455
456 vtpci_describe_features(sc, "negotiated", features);
457 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
458
459 return (features);
460}
461
462static int
463vtpci_with_feature(device_t dev, uint64_t feature)
464{
465 struct vtpci_softc *sc;
466
467 sc = device_get_softc(dev);
468
469 return ((sc->vtpci_features & feature) != 0);
470}
471
472static int
473vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
474 struct vq_alloc_info *vq_info)
475{
476 struct vtpci_softc *sc;
477 struct virtqueue *vq;
478 struct vtpci_virtqueue *vqx;
479 struct vq_alloc_info *info;
480 int idx, error;
481 uint16_t size;
482
483 sc = device_get_softc(dev);
484
485 if (sc->vtpci_nvqs != 0)
486 return (EALREADY);
487 if (nvqs <= 0)
488 return (EINVAL);
489
490 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
491 M_DEVBUF, M_NOWAIT | M_ZERO);
492 if (sc->vtpci_vqs == NULL)
493 return (ENOMEM);
494
495 for (idx = 0; idx < nvqs; idx++) {
496 vqx = &sc->vtpci_vqs[idx];
497 info = &vq_info[idx];
498
499 vtpci_select_virtqueue(sc, idx);
500 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
501
502 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
503 0xFFFFFFFFUL, info, &vq);
504 if (error) {
505 device_printf(dev,
506 "cannot allocate virtqueue %d: %d\n", idx, error);
507 break;
508 }
509
510 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
511 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
512
513 vqx->vtv_vq = *info->vqai_vq = vq;
514 vqx->vtv_no_intr = info->vqai_intr == NULL;
515
516 sc->vtpci_nvqs++;
517 }
518
519 if (error)
520 vtpci_free_virtqueues(sc);
521
522 return (error);
523}
524
525static int
526vtpci_setup_intr(device_t dev, enum intr_type type)
527{
528 struct vtpci_softc *sc;
529 int attempt, error;
530
531 sc = device_get_softc(dev);
532
533 for (attempt = 0; attempt < 5; attempt++) {
534 /*
535 * Start with the most desirable interrupt configuration and
536 * fallback towards less desirable ones.
537 */
538 switch (attempt) {
539 case 0:
540 error = vtpci_alloc_intr_msix_pervq(sc);
541 break;
542 case 1:
543 error = vtpci_alloc_intr_msix_shared(sc);
544 break;
545 case 2:
546 error = vtpci_alloc_intr_msi(sc);
547 break;
548 case 3:
549 error = vtpci_alloc_intr_legacy(sc);
550 break;
551 default:
552 device_printf(dev,
553 "exhausted all interrupt allocation attempts\n");
554 return (ENXIO);
555 }
556
557 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
558 break;
559
560 vtpci_cleanup_setup_intr_attempt(sc);
561 }
562
563 if (bootverbose) {
564 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
565 device_printf(dev, "using legacy interrupt\n");
566 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
567 device_printf(dev, "using MSI interrupt\n");
568 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
569 device_printf(dev, "using shared MSIX interrupts\n");
570 else
571 device_printf(dev, "using per VQ MSIX interrupts\n");
572 }
573
574 return (0);
575}
576
577static void
578vtpci_stop(device_t dev)
579{
580
581 vtpci_reset(device_get_softc(dev));
582}
583
584static int
585vtpci_reinit(device_t dev, uint64_t features)
586{
587 struct vtpci_softc *sc;
588 int idx, error;
589
590 sc = device_get_softc(dev);
591
592 /*
593 * Redrive the device initialization. This is a bit of an abuse of
594 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
595 * play nice.
596 *
597 * We do not allow the host device to change from what was originally
598 * negotiated beyond what the guest driver changed. MSIX state should
599 * not change, number of virtqueues and their size remain the same, etc.
600 * This will need to be rethought when we want to support migration.
601 */
602
603 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
604 vtpci_stop(dev);
605
606 /*
607 * Quickly drive the status through ACK and DRIVER. The device
608 * does not become usable again until vtpci_reinit_complete().
609 */
610 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
611 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
612
613 vtpci_negotiate_features(dev, features);
614
615 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
616 error = vtpci_reinit_virtqueue(sc, idx);
617 if (error)
618 return (error);
619 }
620
621 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
622 error = vtpci_set_host_msix_vectors(sc);
623 if (error)
624 return (error);
625 }
626
627 return (0);
628}
629
630static void
631vtpci_reinit_complete(device_t dev)
632{
633
634 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
635}
636
637static void
638vtpci_notify_virtqueue(device_t dev, uint16_t queue)
639{
640 struct vtpci_softc *sc;
641
642 sc = device_get_softc(dev);
643
644 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
645}
646
647static uint8_t
648vtpci_get_status(device_t dev)
649{
650 struct vtpci_softc *sc;
651
652 sc = device_get_softc(dev);
653
654 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
655}
656
657static void
658vtpci_set_status(device_t dev, uint8_t status)
659{
660 struct vtpci_softc *sc;
661
662 sc = device_get_softc(dev);
663
664 if (status != VIRTIO_CONFIG_STATUS_RESET)
665 status |= vtpci_get_status(dev);
666
667 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
668}
669
670static void
671vtpci_read_dev_config(device_t dev, bus_size_t offset,
672 void *dst, int length)
673{
674 struct vtpci_softc *sc;
675 bus_size_t off;
676 uint8_t *d;
677 int size;
678
679 sc = device_get_softc(dev);
680 off = VIRTIO_PCI_CONFIG(sc) + offset;
681
682 for (d = dst; length > 0; d += size, off += size, length -= size) {
683 if (length >= 4) {
684 size = 4;
685 *(uint32_t *)d = vtpci_read_config_4(sc, off);
686 } else if (length >= 2) {
687 size = 2;
688 *(uint16_t *)d = vtpci_read_config_2(sc, off);
689 } else {
690 size = 1;
691 *d = vtpci_read_config_1(sc, off);
692 }
693 }
694}
695
696static void
697vtpci_write_dev_config(device_t dev, bus_size_t offset,
698 void *src, int length)
699{
700 struct vtpci_softc *sc;
701 bus_size_t off;
702 uint8_t *s;
703 int size;
704
705 sc = device_get_softc(dev);
706 off = VIRTIO_PCI_CONFIG(sc) + offset;
707
708 for (s = src; length > 0; s += size, off += size, length -= size) {
709 if (length >= 4) {
710 size = 4;
711 vtpci_write_config_4(sc, off, *(uint32_t *)s);
712 } else if (length >= 2) {
713 size = 2;
714 vtpci_write_config_2(sc, off, *(uint16_t *)s);
715 } else {
716 size = 1;
717 vtpci_write_config_1(sc, off, *s);
718 }
719 }
720}
721
722static void
723vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
724 uint64_t features)
725{
726 device_t dev, child;
727
728 dev = sc->vtpci_dev;
729 child = sc->vtpci_child_dev;
730
731 if (device_is_attached(child) && bootverbose == 0)
732 return;
733
734 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
735}
736
737static void
738vtpci_probe_and_attach_child(struct vtpci_softc *sc)
739{
740 device_t dev, child;
741
742 dev = sc->vtpci_dev;
743 child = sc->vtpci_child_dev;
744
745 if (child == NULL)
746 return;
747
748 if (device_get_state(child) != DS_NOTPRESENT)
749 return;
750
751 if (device_probe(child) != 0)
752 return;
753
754 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
755 if (device_attach(child) != 0) {
756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
757 vtpci_reset(sc);
758 vtpci_release_child_resources(sc);
759 /* Reset status for future attempt. */
760 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
761 } else {
762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
763 VIRTIO_ATTACH_COMPLETED(child);
764 }
765}
766
767static int
768vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
769{
770 device_t dev;
771 int nmsix, cnt, required;
772
773 dev = sc->vtpci_dev;
774
775 /* Allocate an additional vector for the config changes. */
776 required = nvectors + 1;
777
778 nmsix = pci_msix_count(dev);
779 if (nmsix < required)
780 return (1);
781
782 cnt = required;
783 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
784 sc->vtpci_nmsix_resources = required;
785 return (0);
786 }
787
788 pci_release_msi(dev);
789
790 return (1);
791}
792
793static int
794vtpci_alloc_msi(struct vtpci_softc *sc)
795{
796 device_t dev;
797 int nmsi, cnt, required;
798
799 dev = sc->vtpci_dev;
800 required = 1;
801
802 nmsi = pci_msi_count(dev);
803 if (nmsi < required)
804 return (1);
805
806 cnt = required;
807 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
808 return (0);
809
810 pci_release_msi(dev);
811
812 return (1);
813}
814
815static int
816vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
817{
818 int i, nvectors, error;
819
820 if (vtpci_disable_msix != 0 ||
821 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
822 return (ENOTSUP);
823
824 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
825 if (sc->vtpci_vqs[i].vtv_no_intr == 0)
826 nvectors++;
827 }
828
829 error = vtpci_alloc_msix(sc, nvectors);
830 if (error)
831 return (error);
832
833 sc->vtpci_flags |= VTPCI_FLAG_MSIX;
834
835 return (0);
836}
837
838static int
839vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
840{
841 int error;
842
843 if (vtpci_disable_msix != 0 ||
844 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
845 return (ENOTSUP);
846
847 error = vtpci_alloc_msix(sc, 1);
848 if (error)
849 return (error);
850
851 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
852
853 return (0);
854}
855
856static int
857vtpci_alloc_intr_msi(struct vtpci_softc *sc)
858{
859 int error;
860
861 /* Only BHyVe supports MSI. */
862 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
863 return (ENOTSUP);
864
865 error = vtpci_alloc_msi(sc);
866 if (error)
867 return (error);
868
869 sc->vtpci_flags |= VTPCI_FLAG_MSI;
870
871 return (0);
872}
873
874static int
875vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
876{
877
878 sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
879
880 return (0);
881}
882
883static int
884vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
885 struct vtpci_interrupt *intr)
886{
887 struct resource *irq;
888
889 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
890 if (irq == NULL)
891 return (ENXIO);
892
893 intr->vti_irq = irq;
894 intr->vti_rid = rid;
895
896 return (0);
897}
898
899static int
900vtpci_alloc_intr_resources(struct vtpci_softc *sc)
901{
902 struct vtpci_interrupt *intr;
903 int i, rid, flags, nvq_intrs, error;
904
905 rid = 0;
906 flags = RF_ACTIVE;
907
908 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
909 flags |= RF_SHAREABLE;
910 else
911 rid = 1;
912
913 /*
914 * For legacy and MSI interrupts, this single resource handles all
915 * interrupts. For MSIX, this resource is used for the configuration
916 * changed interrupt.
917 */
918 intr = &sc->vtpci_device_interrupt;
919 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
920 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
921 return (error);
922
923 /* Subtract one for the configuration changed interrupt. */
924 nvq_intrs = sc->vtpci_nmsix_resources - 1;
925
926 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
927 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
928 if (sc->vtpci_msix_vq_interrupts == NULL)
929 return (ENOMEM);
930
931 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
932 error = vtpci_alloc_interrupt(sc, rid, flags, intr);
933 if (error)
934 return (error);
935 }
936
937 return (0);
938}
939
940static int
941vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
942{
943 struct vtpci_interrupt *intr;
944 int error;
945
946 intr = &sc->vtpci_device_interrupt;
947 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
948 vtpci_legacy_intr, sc, &intr->vti_handler);
949
950 return (error);
951}
952
953static int
954vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
955{
956 struct vtpci_virtqueue *vqx;
957 struct vtpci_interrupt *intr;
958 int i, error;
959
960 intr = sc->vtpci_msix_vq_interrupts;
961
962 for (i = 0; i < sc->vtpci_nvqs; i++) {
963 vqx = &sc->vtpci_vqs[i];
964
965 if (vqx->vtv_no_intr)
966 continue;
967
968 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
969 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
970 &intr->vti_handler);
971 if (error)
972 return (error);
973
974 intr++;
975 }
976
977 return (0);
978}
979
980static int
981vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
982{
983 device_t dev;
984 struct vtpci_interrupt *intr;
985 int error;
986
987 dev = sc->vtpci_dev;
988 intr = &sc->vtpci_device_interrupt;
989
990 error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
991 vtpci_config_intr, sc, &intr->vti_handler);
992 if (error)
993 return (error);
994
995 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
996 intr = sc->vtpci_msix_vq_interrupts;
997 error = bus_setup_intr(dev, intr->vti_irq, type,
998 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
999 &intr->vti_handler);
1000 } else
1001 error = vtpci_setup_pervq_msix_interrupts(sc, type);
1002
1003 return (error ? error : vtpci_set_host_msix_vectors(sc));
1004}
1005
1006static int
1007vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1008{
1009 int error;
1010
1011 type |= INTR_MPSAFE;
1012 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1013 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1014
1015 error = vtpci_alloc_intr_resources(sc);
1016 if (error)
1017 return (error);
1018
1019 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1020 error = vtpci_setup_legacy_interrupt(sc, type);
1021 else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1022 error = vtpci_setup_msi_interrupt(sc, type);
1023 else
1024 error = vtpci_setup_msix_interrupts(sc, type);
1025
1026 return (error);
1027}
1028
1029static int
1030vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1031 struct vtpci_interrupt *intr)
1032{
1033 device_t dev;
1034 uint16_t vector;
1035
1036 dev = sc->vtpci_dev;
1037
1038 if (intr != NULL) {
1039 /* Map from guest rid to host vector. */
1040 vector = intr->vti_rid - 1;
1041 } else
1042 vector = VIRTIO_MSI_NO_VECTOR;
1043
1044 vtpci_write_config_2(sc, offset, vector);
1045
1046 /* Read vector to determine if the host had sufficient resources. */
1047 if (vtpci_read_config_2(sc, offset) != vector) {
1048 device_printf(dev,
1049 "insufficient host resources for MSIX interrupts\n");
1050 return (ENODEV);
1051 }
1052
1053 return (0);
1054}
1055
1056static int
1057vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1058{
1059 struct vtpci_interrupt *intr, *tintr;
1060 int idx, offset, error;
1061
1062 intr = &sc->vtpci_device_interrupt;
1063 offset = VIRTIO_MSI_CONFIG_VECTOR;
1064
1065 error = vtpci_register_msix_vector(sc, offset, intr);
1066 if (error)
1067 return (error);
1068
1069 intr = sc->vtpci_msix_vq_interrupts;
1070 offset = VIRTIO_MSI_QUEUE_VECTOR;
1071
1072 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1073 vtpci_select_virtqueue(sc, idx);
1074
1075 if (sc->vtpci_vqs[idx].vtv_no_intr)
1076 tintr = NULL;
1077 else
1078 tintr = intr;
1079
1080 error = vtpci_register_msix_vector(sc, offset, tintr);
1081 if (error)
1082 break;
1083
1084 /*
1085 * For shared MSIX, all the virtqueues share the first
1086 * interrupt.
1087 */
1088 if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1089 intr++;
1090 }
1091
1092 return (error);
1093}
1094
1095static int
1096vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1097{
1098 struct vtpci_virtqueue *vqx;
1099 struct virtqueue *vq;
1100 int error;
1101 uint16_t size;
1102
1103 vqx = &sc->vtpci_vqs[idx];
1104 vq = vqx->vtv_vq;
1105
1106 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1107
1108 vtpci_select_virtqueue(sc, idx);
1109 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1110
1111 error = virtqueue_reinit(vq, size);
1112 if (error)
1113 return (error);
1114
1115 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1116 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1117
1118 return (0);
1119}
1120
1121static void
1122vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1123{
1124 device_t dev;
1125
1126 dev = sc->vtpci_dev;
1127
1128 if (intr->vti_handler != NULL) {
1129 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1130 intr->vti_handler = NULL;
1131 }
1132
1133 if (intr->vti_irq != NULL) {
1134 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1135 intr->vti_irq);
1136 intr->vti_irq = NULL;
1137 intr->vti_rid = -1;
1138 }
1139}
1140
1141static void
1142vtpci_free_interrupts(struct vtpci_softc *sc)
1143{
1144 struct vtpci_interrupt *intr;
1145 int i, nvq_intrs;
1146
1147 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1148
1149 if (sc->vtpci_nmsix_resources != 0) {
1150 nvq_intrs = sc->vtpci_nmsix_resources - 1;
1151 sc->vtpci_nmsix_resources = 0;
1152
1153 intr = sc->vtpci_msix_vq_interrupts;
1154 if (intr != NULL) {
1155 for (i = 0; i < nvq_intrs; i++, intr++)
1156 vtpci_free_interrupt(sc, intr);
1157
1158 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1159 sc->vtpci_msix_vq_interrupts = NULL;
1160 }
1161 }
1162
1163 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1164 pci_release_msi(sc->vtpci_dev);
1165
1166 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1167}
1168
1169static void
1170vtpci_free_virtqueues(struct vtpci_softc *sc)
1171{
1172 struct vtpci_virtqueue *vqx;
1173 int idx;
1174
1175 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1176 vqx = &sc->vtpci_vqs[idx];
1177
1178 vtpci_select_virtqueue(sc, idx);
1179 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1180
1181 virtqueue_free(vqx->vtv_vq);
1182 vqx->vtv_vq = NULL;
1183 }
1184
1185 free(sc->vtpci_vqs, M_DEVBUF);
1186 sc->vtpci_vqs = NULL;
1187 sc->vtpci_nvqs = 0;
1188}
1189
1190static void
1191vtpci_release_child_resources(struct vtpci_softc *sc)
1192{
1193
1194 vtpci_free_interrupts(sc);
1195 vtpci_free_virtqueues(sc);
1196}
1197
1198static void
1199vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1200{
1201 int idx;
1202
1203 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1204 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1205 VIRTIO_MSI_NO_VECTOR);
1206
1207 for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1208 vtpci_select_virtqueue(sc, idx);
1209 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1210 VIRTIO_MSI_NO_VECTOR);
1211 }
1212 }
1213
1214 vtpci_free_interrupts(sc);
1215}
1216
1217static void
1218vtpci_reset(struct vtpci_softc *sc)
1219{
1220
1221 /*
1222 * Setting the status to RESET sets the host device to
1223 * the original, uninitialized state.
1224 */
1225 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1226}
1227
1228static void
1229vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1230{
1231
1232 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1233}
1234
1235static void
1236vtpci_legacy_intr(void *xsc)
1237{
1238 struct vtpci_softc *sc;
1239 struct vtpci_virtqueue *vqx;
1240 int i;
1241 uint8_t isr;
1242
1243 sc = xsc;
1244 vqx = &sc->vtpci_vqs[0];
1245
1246 /* Reading the ISR also clears it. */
1247 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1248
1249 if (isr & VIRTIO_PCI_ISR_CONFIG)
1250 vtpci_config_intr(sc);
1251
1252 if (isr & VIRTIO_PCI_ISR_INTR) {
1253 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1254 if (vqx->vtv_no_intr == 0)
1255 virtqueue_intr(vqx->vtv_vq);
1256 }
1257 }
1258}
1259
1260static int
1261vtpci_vq_shared_intr_filter(void *xsc)
1262{
1263 struct vtpci_softc *sc;
1264 struct vtpci_virtqueue *vqx;
1265 int i, rc;
1266
1267 rc = 0;
1268 sc = xsc;
1269 vqx = &sc->vtpci_vqs[0];
1270
1271 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1272 if (vqx->vtv_no_intr == 0)
1273 rc |= virtqueue_intr_filter(vqx->vtv_vq);
1274 }
1275
1276 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1277}
1278
1279static void
1280vtpci_vq_shared_intr(void *xsc)
1281{
1282 struct vtpci_softc *sc;
1283 struct vtpci_virtqueue *vqx;
1284 int i;
1285
1286 sc = xsc;
1287 vqx = &sc->vtpci_vqs[0];
1288
1289 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1290 if (vqx->vtv_no_intr == 0)
1291 virtqueue_intr(vqx->vtv_vq);
1292 }
1293}
1294
1295static int
1296vtpci_vq_intr_filter(void *xvq)
1297{
1298 struct virtqueue *vq;
1299 int rc;
1300
1301 vq = xvq;
1302 rc = virtqueue_intr_filter(vq);
1303
1304 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1305}
1306
1307static void
1308vtpci_vq_intr(void *xvq)
1309{
1310 struct virtqueue *vq;
1311
1312 vq = xvq;
1313 virtqueue_intr(vq);
1314}
1315
1316static void
1317vtpci_config_intr(void *xsc)
1318{
1319 struct vtpci_softc *sc;
1320 device_t child;
1321
1322 sc = xsc;
1323 child = sc->vtpci_child_dev;
1324
1325 if (child != NULL)
1326 VIRTIO_CONFIG_CHANGE(child);
1327}