1/*-
2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8 * ("CTSRD"), as part of the DARPA CRASH research programme.
9 *
10 * Portions of this software were developed by Andrew Turner
11 * under sponsorship from the FreeBSD Foundation.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35/*
36 * VirtIO MMIO interface.
37 * This driver is heavily based on VirtIO PCI interface driver.
38 */
39
40/*
41 * FDT example:
42 *		virtio_block@1000 {
43 *			compatible = "virtio,mmio";
44 *			reg = <0x1000 0x100>;
45 *			interrupts = <63>;
46 *			interrupt-parent = <&GIC>;
47 *		};
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD$");
52
53#include <sys/param.h>
54#include <sys/systm.h>
55#include <sys/bus.h>
56#include <sys/kernel.h>
57#include <sys/module.h>
58#include <sys/malloc.h>
59#include <sys/rman.h>
60
61#include <machine/bus.h>
62#include <machine/resource.h>
63
64#include <dev/fdt/fdt_common.h>
65#include <dev/ofw/openfirm.h>
66#include <dev/ofw/ofw_bus.h>
67#include <dev/ofw/ofw_bus_subr.h>
68
69#include <dev/virtio/virtio.h>
70#include <dev/virtio/virtqueue.h>
71#include <dev/virtio/mmio/virtio_mmio.h>
72
73#include "virtio_mmio_if.h"
74#include "virtio_bus_if.h"
75#include "virtio_if.h"
76
77#define	PAGE_SHIFT	12
78
79struct vtmmio_virtqueue {
80	struct virtqueue	*vtv_vq;
81	int			 vtv_no_intr;
82};
83
84struct vtmmio_softc {
85	device_t			dev;
86	device_t			platform;
87	struct resource			*res[2];
88
89	uint64_t			vtmmio_features;
90	uint32_t			vtmmio_flags;
91
92	/* This "bus" will only ever have one child. */
93	device_t			vtmmio_child_dev;
94	struct virtio_feature_desc	*vtmmio_child_feat_desc;
95
96	int				vtmmio_nvqs;
97	struct vtmmio_virtqueue		*vtmmio_vqs;
98	void				*ih;
99};
100
101static int	vtmmio_probe(device_t);
102static int	vtmmio_attach(device_t);
103static int	vtmmio_detach(device_t);
104static int	vtmmio_suspend(device_t);
105static int	vtmmio_resume(device_t);
106static int	vtmmio_shutdown(device_t);
107static void	vtmmio_driver_added(device_t, driver_t *);
108static void	vtmmio_child_detached(device_t, device_t);
109static int	vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
110static int	vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
111static uint64_t	vtmmio_negotiate_features(device_t, uint64_t);
112static int	vtmmio_with_feature(device_t, uint64_t);
113static int	vtmmio_alloc_virtqueues(device_t, int, int,
114		    struct vq_alloc_info *);
115static int	vtmmio_setup_intr(device_t, enum intr_type);
116static void	vtmmio_stop(device_t);
117static void	vtmmio_poll(device_t);
118static int	vtmmio_reinit(device_t, uint64_t);
119static void	vtmmio_reinit_complete(device_t);
120static void	vtmmio_notify_virtqueue(device_t, uint16_t);
121static uint8_t	vtmmio_get_status(device_t);
122static void	vtmmio_set_status(device_t, uint8_t);
123static void	vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
124static void	vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
125static void	vtmmio_describe_features(struct vtmmio_softc *, const char *,
126		    uint64_t);
127static void	vtmmio_probe_and_attach_child(struct vtmmio_softc *);
128static int	vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
129static void	vtmmio_free_interrupts(struct vtmmio_softc *);
130static void	vtmmio_free_virtqueues(struct vtmmio_softc *);
131static void	vtmmio_release_child_resources(struct vtmmio_softc *);
132static void	vtmmio_reset(struct vtmmio_softc *);
133static void	vtmmio_select_virtqueue(struct vtmmio_softc *, int);
134static void	vtmmio_vq_intr(void *);
135
136/*
137 * I/O port read/write wrappers.
138 */
139#define vtmmio_write_config_1(sc, o, v)				\
140do {								\
141	if (sc->platform != NULL)				\
142		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
143	bus_write_1((sc)->res[0], (o), (v)); 			\
144	if (sc->platform != NULL)				\
145		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
146} while (0)
147#define vtmmio_write_config_2(sc, o, v)				\
148do {								\
149	if (sc->platform != NULL)				\
150		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
151	bus_write_2((sc)->res[0], (o), (v));			\
152	if (sc->platform != NULL)				\
153		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
154} while (0)
155#define vtmmio_write_config_4(sc, o, v)				\
156do {								\
157	if (sc->platform != NULL)				\
158		VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v));	\
159	bus_write_4((sc)->res[0], (o), (v));			\
160	if (sc->platform != NULL)				\
161		VIRTIO_MMIO_NOTE(sc->platform, (o), (v));	\
162} while (0)
163
164#define vtmmio_read_config_1(sc, o) \
165	bus_read_1((sc)->res[0], (o))
166#define vtmmio_read_config_2(sc, o) \
167	bus_read_2((sc)->res[0], (o))
168#define vtmmio_read_config_4(sc, o) \
169	bus_read_4((sc)->res[0], (o))
170
171static device_method_t vtmmio_methods[] = {
172	/* Device interface. */
173	DEVMETHOD(device_probe,			  vtmmio_probe),
174	DEVMETHOD(device_attach,		  vtmmio_attach),
175	DEVMETHOD(device_detach,		  vtmmio_detach),
176	DEVMETHOD(device_suspend,		  vtmmio_suspend),
177	DEVMETHOD(device_resume,		  vtmmio_resume),
178	DEVMETHOD(device_shutdown,		  vtmmio_shutdown),
179
180	/* Bus interface. */
181	DEVMETHOD(bus_driver_added,		  vtmmio_driver_added),
182	DEVMETHOD(bus_child_detached,		  vtmmio_child_detached),
183	DEVMETHOD(bus_read_ivar,		  vtmmio_read_ivar),
184	DEVMETHOD(bus_write_ivar,		  vtmmio_write_ivar),
185
186	/* VirtIO bus interface. */
187	DEVMETHOD(virtio_bus_negotiate_features,  vtmmio_negotiate_features),
188	DEVMETHOD(virtio_bus_with_feature,	  vtmmio_with_feature),
189	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtmmio_alloc_virtqueues),
190	DEVMETHOD(virtio_bus_setup_intr,	  vtmmio_setup_intr),
191	DEVMETHOD(virtio_bus_stop,		  vtmmio_stop),
192	DEVMETHOD(virtio_bus_poll,		  vtmmio_poll),
193	DEVMETHOD(virtio_bus_reinit,		  vtmmio_reinit),
194	DEVMETHOD(virtio_bus_reinit_complete,	  vtmmio_reinit_complete),
195	DEVMETHOD(virtio_bus_notify_vq,		  vtmmio_notify_virtqueue),
196	DEVMETHOD(virtio_bus_read_device_config,  vtmmio_read_dev_config),
197	DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
198
199	DEVMETHOD_END
200};
201
202static driver_t vtmmio_driver = {
203	"virtio_mmio",
204	vtmmio_methods,
205	sizeof(struct vtmmio_softc)
206};
207
208devclass_t vtmmio_devclass;
209
210DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
211DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
212MODULE_VERSION(virtio_mmio, 1);
213MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
214MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
215
216static int
217vtmmio_setup_intr(device_t dev, enum intr_type type)
218{
219	struct vtmmio_softc *sc;
220	int rid;
221	int err;
222
223	sc = device_get_softc(dev);
224
225	if (sc->platform != NULL) {
226		err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
227					vtmmio_vq_intr, sc);
228		if (err == 0) {
229			/* Okay we have backend-specific interrupts */
230			return (0);
231		}
232	}
233
234	rid = 0;
235	sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
236		RF_ACTIVE);
237	if (!sc->res[1]) {
238		device_printf(dev, "Can't allocate interrupt\n");
239		return (ENXIO);
240	}
241
242	if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
243		NULL, vtmmio_vq_intr, sc, &sc->ih)) {
244		device_printf(dev, "Can't setup the interrupt\n");
245		return (ENXIO);
246	}
247
248	return (0);
249}
250
251static int
252vtmmio_probe(device_t dev)
253{
254
255	if (!ofw_bus_status_okay(dev))
256		return (ENXIO);
257
258	if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
259		return (ENXIO);
260
261	device_set_desc(dev, "VirtIO MMIO adapter");
262	return (BUS_PROBE_DEFAULT);
263}
264
265static int
266vtmmio_setup_platform(struct vtmmio_softc *sc)
267{
268	phandle_t platform_node;
269	struct fdt_ic *ic;
270	phandle_t xref;
271	phandle_t node;
272
273	sc->platform = NULL;
274
275	if ((node = ofw_bus_get_node(sc->dev)) == -1)
276		return (ENXIO);
277
278	if (OF_searchencprop(node, "platform", &xref,
279		sizeof(xref)) == -1) {
280		return (ENXIO);
281	}
282
283	platform_node = OF_node_from_xref(xref);
284
285	SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
286		if (ic->iph == platform_node) {
287			sc->platform = ic->dev;
288			break;
289		}
290	}
291
292	if (sc->platform == NULL) {
293		/* No platform-specific device. Ignore it. */
294	}
295
296	return (0);
297}
298
299static int
300vtmmio_attach(device_t dev)
301{
302	struct vtmmio_softc *sc;
303	device_t child;
304	int rid;
305
306	sc = device_get_softc(dev);
307	sc->dev = dev;
308
309	vtmmio_setup_platform(sc);
310
311	rid = 0;
312	sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
313			RF_ACTIVE);
314	if (!sc->res[0]) {
315		device_printf(dev, "Cannot allocate memory window.\n");
316		return (ENXIO);
317	}
318
319	vtmmio_reset(sc);
320
321	/* Tell the host we've noticed this device. */
322	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
323
324	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
325		device_printf(dev, "Cannot create child device.\n");
326		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
327		vtmmio_detach(dev);
328		return (ENOMEM);
329	}
330
331	sc->vtmmio_child_dev = child;
332	vtmmio_probe_and_attach_child(sc);
333
334	return (0);
335}
336
337static int
338vtmmio_detach(device_t dev)
339{
340	struct vtmmio_softc *sc;
341	device_t child;
342	int error;
343
344	sc = device_get_softc(dev);
345
346	if ((child = sc->vtmmio_child_dev) != NULL) {
347		error = device_delete_child(dev, child);
348		if (error)
349			return (error);
350		sc->vtmmio_child_dev = NULL;
351	}
352
353	vtmmio_reset(sc);
354
355	if (sc->res[0] != NULL) {
356		bus_release_resource(dev, SYS_RES_MEMORY, 0,
357		    sc->res[0]);
358		sc->res[0] = NULL;
359	}
360
361	return (0);
362}
363
364static int
365vtmmio_suspend(device_t dev)
366{
367
368	return (bus_generic_suspend(dev));
369}
370
371static int
372vtmmio_resume(device_t dev)
373{
374
375	return (bus_generic_resume(dev));
376}
377
378static int
379vtmmio_shutdown(device_t dev)
380{
381
382	(void) bus_generic_shutdown(dev);
383
384	/* Forcibly stop the host device. */
385	vtmmio_stop(dev);
386
387	return (0);
388}
389
390static void
391vtmmio_driver_added(device_t dev, driver_t *driver)
392{
393	struct vtmmio_softc *sc;
394
395	sc = device_get_softc(dev);
396
397	vtmmio_probe_and_attach_child(sc);
398}
399
400static void
401vtmmio_child_detached(device_t dev, device_t child)
402{
403	struct vtmmio_softc *sc;
404
405	sc = device_get_softc(dev);
406
407	vtmmio_reset(sc);
408	vtmmio_release_child_resources(sc);
409}
410
411static int
412vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
413{
414	struct vtmmio_softc *sc;
415
416	sc = device_get_softc(dev);
417
418	if (sc->vtmmio_child_dev != child)
419		return (ENOENT);
420
421	switch (index) {
422	case VIRTIO_IVAR_DEVTYPE:
423	case VIRTIO_IVAR_SUBDEVICE:
424		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
425		break;
426	case VIRTIO_IVAR_VENDOR:
427		*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
428		break;
429	default:
430		return (ENOENT);
431	}
432
433	return (0);
434}
435
436static int
437vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
438{
439	struct vtmmio_softc *sc;
440
441	sc = device_get_softc(dev);
442
443	if (sc->vtmmio_child_dev != child)
444		return (ENOENT);
445
446	switch (index) {
447	case VIRTIO_IVAR_FEATURE_DESC:
448		sc->vtmmio_child_feat_desc = (void *) value;
449		break;
450	default:
451		return (ENOENT);
452	}
453
454	return (0);
455}
456
457static uint64_t
458vtmmio_negotiate_features(device_t dev, uint64_t child_features)
459{
460	struct vtmmio_softc *sc;
461	uint64_t host_features, features;
462
463	sc = device_get_softc(dev);
464
465	host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
466	vtmmio_describe_features(sc, "host", host_features);
467
468	/*
469	 * Limit negotiated features to what the driver, virtqueue, and
470	 * host all support.
471	 */
472	features = host_features & child_features;
473	features = virtqueue_filter_features(features);
474	sc->vtmmio_features = features;
475
476	vtmmio_describe_features(sc, "negotiated", features);
477	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
478
479	return (features);
480}
481
482static int
483vtmmio_with_feature(device_t dev, uint64_t feature)
484{
485	struct vtmmio_softc *sc;
486
487	sc = device_get_softc(dev);
488
489	return ((sc->vtmmio_features & feature) != 0);
490}
491
492static int
493vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
494    struct vq_alloc_info *vq_info)
495{
496	struct vtmmio_virtqueue *vqx;
497	struct vq_alloc_info *info;
498	struct vtmmio_softc *sc;
499	struct virtqueue *vq;
500	uint32_t size;
501	int idx, error;
502
503	sc = device_get_softc(dev);
504
505	if (sc->vtmmio_nvqs != 0)
506		return (EALREADY);
507	if (nvqs <= 0)
508		return (EINVAL);
509
510	sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
511	    M_DEVBUF, M_NOWAIT | M_ZERO);
512	if (sc->vtmmio_vqs == NULL)
513		return (ENOMEM);
514
515	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
516	    (1 << PAGE_SHIFT));
517
518	for (idx = 0; idx < nvqs; idx++) {
519		vqx = &sc->vtmmio_vqs[idx];
520		info = &vq_info[idx];
521
522		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
523
524		vtmmio_select_virtqueue(sc, idx);
525		size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
526
527		error = virtqueue_alloc(dev, idx, size,
528		    VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
529		if (error) {
530			device_printf(dev,
531			    "cannot allocate virtqueue %d: %d\n",
532			    idx, error);
533			break;
534		}
535
536		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
537		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
538		    VIRTIO_MMIO_VRING_ALIGN);
539#if 0
540		device_printf(dev, "virtqueue paddr 0x%08lx\n",
541		    (uint64_t)virtqueue_paddr(vq));
542#endif
543		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
544		    virtqueue_paddr(vq) >> PAGE_SHIFT);
545
546		vqx->vtv_vq = *info->vqai_vq = vq;
547		vqx->vtv_no_intr = info->vqai_intr == NULL;
548
549		sc->vtmmio_nvqs++;
550	}
551
552	if (error)
553		vtmmio_free_virtqueues(sc);
554
555	return (error);
556}
557
558static void
559vtmmio_stop(device_t dev)
560{
561
562	vtmmio_reset(device_get_softc(dev));
563}
564
565static void
566vtmmio_poll(device_t dev)
567{
568	struct vtmmio_softc *sc;
569
570	sc = device_get_softc(dev);
571
572	if (sc->platform != NULL)
573		VIRTIO_MMIO_POLL(sc->platform);
574}
575
576static int
577vtmmio_reinit(device_t dev, uint64_t features)
578{
579	struct vtmmio_softc *sc;
580	int idx, error;
581
582	sc = device_get_softc(dev);
583
584	if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
585		vtmmio_stop(dev);
586
587	/*
588	 * Quickly drive the status through ACK and DRIVER. The device
589	 * does not become usable again until vtmmio_reinit_complete().
590	 */
591	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
592	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
593
594	vtmmio_negotiate_features(dev, features);
595
596	vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE,
597	    (1 << PAGE_SHIFT));
598
599	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
600		error = vtmmio_reinit_virtqueue(sc, idx);
601		if (error)
602			return (error);
603	}
604
605	return (0);
606}
607
608static void
609vtmmio_reinit_complete(device_t dev)
610{
611
612	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
613}
614
615static void
616vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
617{
618	struct vtmmio_softc *sc;
619
620	sc = device_get_softc(dev);
621
622	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
623}
624
625static uint8_t
626vtmmio_get_status(device_t dev)
627{
628	struct vtmmio_softc *sc;
629
630	sc = device_get_softc(dev);
631
632	return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
633}
634
635static void
636vtmmio_set_status(device_t dev, uint8_t status)
637{
638	struct vtmmio_softc *sc;
639
640	sc = device_get_softc(dev);
641
642	if (status != VIRTIO_CONFIG_STATUS_RESET)
643		status |= vtmmio_get_status(dev);
644
645	vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
646}
647
648static void
649vtmmio_read_dev_config(device_t dev, bus_size_t offset,
650    void *dst, int length)
651{
652	struct vtmmio_softc *sc;
653	bus_size_t off;
654	uint8_t *d;
655	int size;
656
657	sc = device_get_softc(dev);
658	off = VIRTIO_MMIO_CONFIG + offset;
659
660	for (d = dst; length > 0; d += size, off += size, length -= size) {
661#ifdef ALLOW_WORD_ALIGNED_ACCESS
662		if (length >= 4) {
663			size = 4;
664			*(uint32_t *)d = vtmmio_read_config_4(sc, off);
665		} else if (length >= 2) {
666			size = 2;
667			*(uint16_t *)d = vtmmio_read_config_2(sc, off);
668		} else
669#endif
670		{
671			size = 1;
672			*d = vtmmio_read_config_1(sc, off);
673		}
674	}
675}
676
677static void
678vtmmio_write_dev_config(device_t dev, bus_size_t offset,
679    void *src, int length)
680{
681	struct vtmmio_softc *sc;
682	bus_size_t off;
683	uint8_t *s;
684	int size;
685
686	sc = device_get_softc(dev);
687	off = VIRTIO_MMIO_CONFIG + offset;
688
689	for (s = src; length > 0; s += size, off += size, length -= size) {
690#ifdef ALLOW_WORD_ALIGNED_ACCESS
691		if (length >= 4) {
692			size = 4;
693			vtmmio_write_config_4(sc, off, *(uint32_t *)s);
694		} else if (length >= 2) {
695			size = 2;
696			vtmmio_write_config_2(sc, off, *(uint16_t *)s);
697		} else
698#endif
699		{
700			size = 1;
701			vtmmio_write_config_1(sc, off, *s);
702		}
703	}
704}
705
706static void
707vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
708    uint64_t features)
709{
710	device_t dev, child;
711
712	dev = sc->dev;
713	child = sc->vtmmio_child_dev;
714
715	if (device_is_attached(child) || bootverbose == 0)
716		return;
717
718	virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
719}
720
721static void
722vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
723{
724	device_t dev, child;
725
726	dev = sc->dev;
727	child = sc->vtmmio_child_dev;
728
729	if (child == NULL)
730		return;
731
732	if (device_get_state(child) != DS_NOTPRESENT) {
733		return;
734	}
735
736	if (device_probe(child) != 0) {
737		return;
738	}
739
740	vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
741	if (device_attach(child) != 0) {
742		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
743		vtmmio_reset(sc);
744		vtmmio_release_child_resources(sc);
745		/* Reset status for future attempt. */
746		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
747	} else {
748		vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
749		VIRTIO_ATTACH_COMPLETED(child);
750	}
751}
752
753static int
754vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
755{
756	struct vtmmio_virtqueue *vqx;
757	struct virtqueue *vq;
758	int error;
759	uint16_t size;
760
761	vqx = &sc->vtmmio_vqs[idx];
762	vq = vqx->vtv_vq;
763
764	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
765
766	vtmmio_select_virtqueue(sc, idx);
767	size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
768
769	error = virtqueue_reinit(vq, size);
770	if (error)
771		return (error);
772
773	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
774	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
775	    VIRTIO_MMIO_VRING_ALIGN);
776#if 0
777	device_printf(sc->dev, "virtqueue paddr 0x%08lx\n",
778	    (uint64_t)virtqueue_paddr(vq));
779#endif
780	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
781	    virtqueue_paddr(vq) >> PAGE_SHIFT);
782
783	return (0);
784}
785
786static void
787vtmmio_free_interrupts(struct vtmmio_softc *sc)
788{
789
790	if (sc->ih != NULL)
791		bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
792
793	if (sc->res[1] != NULL)
794		bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
795}
796
797static void
798vtmmio_free_virtqueues(struct vtmmio_softc *sc)
799{
800	struct vtmmio_virtqueue *vqx;
801	int idx;
802
803	for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
804		vqx = &sc->vtmmio_vqs[idx];
805
806		vtmmio_select_virtqueue(sc, idx);
807		vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
808
809		virtqueue_free(vqx->vtv_vq);
810		vqx->vtv_vq = NULL;
811	}
812
813	free(sc->vtmmio_vqs, M_DEVBUF);
814	sc->vtmmio_vqs = NULL;
815	sc->vtmmio_nvqs = 0;
816}
817
818static void
819vtmmio_release_child_resources(struct vtmmio_softc *sc)
820{
821
822	vtmmio_free_interrupts(sc);
823	vtmmio_free_virtqueues(sc);
824}
825
826static void
827vtmmio_reset(struct vtmmio_softc *sc)
828{
829
830	/*
831	 * Setting the status to RESET sets the host device to
832	 * the original, uninitialized state.
833	 */
834	vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
835}
836
837static void
838vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
839{
840
841	vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
842}
843
844static void
845vtmmio_vq_intr(void *arg)
846{
847	struct vtmmio_virtqueue *vqx;
848	struct vtmmio_softc *sc;
849	struct virtqueue *vq;
850	uint32_t status;
851	int idx;
852
853	sc = arg;
854
855	status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
856	vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
857
858	/* The config changed */
859	if (status & VIRTIO_MMIO_INT_CONFIG)
860		if (sc->vtmmio_child_dev != NULL)
861			VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
862
863	/* Notify all virtqueues. */
864	if (status & VIRTIO_MMIO_INT_VRING) {
865		for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
866			vqx = &sc->vtmmio_vqs[idx];
867			if (vqx->vtv_no_intr == 0) {
868				vq = vqx->vtv_vq;
869				virtqueue_intr(vq);
870			}
871		}
872	}
873}
874