virtio.c revision 1.2 
1/*	$NetBSD: virtio.c,v 1.2 2011/11/02 15:18:46 jakllsch Exp $	*/
2
3/*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.2 2011/11/02 15:18:46 jakllsch Exp $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/atomic.h>
35#include <sys/bus.h>
36#include <sys/device.h>
37#include <sys/kmem.h>
38
39#include <dev/pci/pcidevs.h>
40#include <dev/pci/pcireg.h>
41#include <dev/pci/pcivar.h>
42
43#include <dev/pci/virtioreg.h>
44#include <dev/pci/virtiovar.h>
45
46#define MINSEG_INDIRECT		2 /* use indirect if nsegs >= this value */
47
48static int	virtio_match(device_t, cfdata_t, void *);
49static void	virtio_attach(device_t, device_t, void *);
50static int	virtio_detach(device_t, int);
51static int	virtio_intr(void *arg);
52static void	virtio_init_vq(struct virtio_softc *,
53		    struct virtqueue *, const bool);
54
55CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc),
56    virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL,
57    DVF_DETACH_SHUTDOWN);
58
59static void
60virtio_set_status(struct virtio_softc *sc, int status)
61{
62	int old = 0;
63
64	if (status != 0)
65		old = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
66				       VIRTIO_CONFIG_DEVICE_STATUS);
67	bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS,
68			  status|old);
69}
70
71#define virtio_device_reset(sc)	virtio_set_status((sc), 0)
72
73static int
74virtio_match(device_t parent, cfdata_t match, void *aux)
75{
76	struct pci_attach_args *pa;
77
78	pa = (struct pci_attach_args *)aux;
79	switch (PCI_VENDOR(pa->pa_id)) {
80	case PCI_VENDOR_QUMRANET:
81		if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
82		     PCI_PRODUCT(pa->pa_id)) &&
83		    (PCI_PRODUCT(pa->pa_id) <=
84		     PCI_PRODUCT_QUMRANET_VIRTIO_103f))
85			return 1;
86		break;
87	}
88
89	return 0;
90}
91
92static const char *virtio_device_name[] = {
93	"Unknown (0)",		/* 0 */
94	"Network",		/* 1 */
95	"Block",		/* 2 */
96	"Console",		/* 3 */
97	"Entropy",		/* 4 */
98	"Memory Balloon",	/* 5 */
99	"Unknown (6)",		/* 6 */
100	"Unknown (7)",		/* 7 */
101	"Unknown (8)",		/* 8 */
102	"9P Transport"		/* 9 */
103};
104#define NDEVNAMES	(sizeof(virtio_device_name)/sizeof(char*))
105
106static void
107virtio_attach(device_t parent, device_t self, void *aux)
108{
109	struct virtio_softc *sc = device_private(self);
110	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
111	pci_chipset_tag_t pc = pa->pa_pc;
112	pcitag_t tag = pa->pa_tag;
113	int revision;
114	pcireg_t id;
115	char const *intrstr;
116	pci_intr_handle_t ih;
117
118	revision = PCI_REVISION(pa->pa_class);
119	if (revision != 0) {
120		aprint_normal(": unknown revision 0x%02x; giving up\n",
121			      revision);
122		return;
123	}
124	aprint_normal("\n");
125	aprint_naive("\n");
126
127	/* subsystem ID shows what I am */
128	id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
129	aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
130			  (PCI_PRODUCT(id) < NDEVNAMES?
131			   virtio_device_name[PCI_PRODUCT(id)] : "Unknown"),
132			  revision);
133
134	sc->sc_dev = self;
135	sc->sc_pc = pc;
136	sc->sc_tag = tag;
137	sc->sc_iot = pa->pa_iot;
138	sc->sc_dmat = pa->pa_dmat;
139	sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
140
141	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
142			   &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) {
143		aprint_error_dev(self, "can't map i/o space\n");
144		return;
145	}
146
147	virtio_device_reset(sc);
148	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
149	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
150
151	/* XXX: use softc as aux... */
152	sc->sc_childdevid = PCI_PRODUCT(id);
153	sc->sc_child = NULL;
154	config_found(self, sc, NULL);
155	if (sc->sc_child == NULL) {
156		aprint_error_dev(self,
157				 "no matching child driver; not configured\n");
158		return;
159	}
160	if (sc->sc_child == (void*)1) { /* this shows error */
161		aprint_error_dev(self,
162				 "virtio configuration failed\n");
163		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
164		return;
165	}
166
167	if (pci_intr_map(pa, &ih)) {
168		aprint_error_dev(self, "couldn't map interrupt\n");
169		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
170		return;
171	}
172	intrstr = pci_intr_string(pc, ih);
173	sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc);
174	if (sc->sc_ih == NULL) {
175		aprint_error_dev(self, "couldn't establish interrupt");
176		if (intrstr != NULL)
177			aprint_error(" at %s", intrstr);
178		aprint_error("\n");
179		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
180		return;
181	}
182	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
183
184	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
185
186	return;
187}
188
189static int
190virtio_detach(device_t self, int flags)
191{
192	struct virtio_softc *sc = device_private(self);
193	int r;
194
195	if (sc->sc_child != 0 && sc->sc_child != (void*)1) {
196		r = config_detach(sc->sc_child, flags);
197		if (r)
198			return r;
199	}
200	KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1);
201	KASSERT(sc->sc_vqs == 0);
202	pci_intr_disestablish(sc->sc_pc, sc->sc_ih);
203	sc->sc_ih = 0;
204	if (sc->sc_iosize)
205		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
206	sc->sc_iosize = 0;
207
208	return 0;
209}
210
211/*
212 * Reset the device.
213 */
214/*
215 * To reset the device to a known state, do following:
216 *	virtio_reset(sc);	     // this will stop the device activity
217 *	<dequeue finished requests>; // virtio_dequeue() still can be called
218 *	<revoke pending requests in the vqs if any>;
219 *	virtio_reinit_begin(sc);     // dequeue prohibitted
220 *	newfeatures = virtio_negotiate_features(sc, requestedfeatures);
221 *	<some other initialization>;
222 *	virtio_reinit_end(sc);	     // device activated; enqueue allowed
223 * Once attached, feature negotiation can only be allowed after virtio_reset.
224 */
225void
226virtio_reset(struct virtio_softc *sc)
227{
228	virtio_device_reset(sc);
229}
230
231void
232virtio_reinit_start(struct virtio_softc *sc)
233{
234	int i;
235
236	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
237	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
238	for (i = 0; i < sc->sc_nvqs; i++) {
239		int n;
240		struct virtqueue *vq = &sc->sc_vqs[i];
241		bus_space_write_2(sc->sc_iot, sc->sc_ioh,
242				  VIRTIO_CONFIG_QUEUE_SELECT,
243				  vq->vq_index);
244		n = bus_space_read_2(sc->sc_iot, sc->sc_ioh,
245				     VIRTIO_CONFIG_QUEUE_SIZE);
246		if (n == 0)	/* vq disappeared */
247			continue;
248		if (n != vq->vq_num) {
249			panic("%s: virtqueue size changed, vq index %d\n",
250			      device_xname(sc->sc_dev),
251			      vq->vq_index);
252		}
253		virtio_init_vq(sc, vq, true);
254		bus_space_write_4(sc->sc_iot, sc->sc_ioh,
255				  VIRTIO_CONFIG_QUEUE_ADDRESS,
256				  (vq->vq_dmamap->dm_segs[0].ds_addr
257				   / VIRTIO_PAGE_SIZE));
258	}
259}
260
261void
262virtio_reinit_end(struct virtio_softc *sc)
263{
264	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
265}
266
267/*
268 * Feature negotiation.
269 */
270uint32_t
271virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
272{
273	uint32_t r;
274
275	if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
276	    !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
277		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
278	r = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
279			     VIRTIO_CONFIG_DEVICE_FEATURES);
280	r &= guest_features;
281	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
282			  VIRTIO_CONFIG_GUEST_FEATURES, r);
283	sc->sc_features = r;
284	if (r & VIRTIO_F_RING_INDIRECT_DESC)
285		sc->sc_indirect = true;
286	else
287		sc->sc_indirect = false;
288
289	return r;
290}
291
292/*
293 * Device configuration registers.
294 */
295uint8_t
296virtio_read_device_config_1(struct virtio_softc *sc, int index)
297{
298	return bus_space_read_1(sc->sc_iot, sc->sc_ioh,
299				sc->sc_config_offset + index);
300}
301
302uint16_t
303virtio_read_device_config_2(struct virtio_softc *sc, int index)
304{
305	return bus_space_read_2(sc->sc_iot, sc->sc_ioh,
306				sc->sc_config_offset + index);
307}
308
309uint32_t
310virtio_read_device_config_4(struct virtio_softc *sc, int index)
311{
312	return bus_space_read_4(sc->sc_iot, sc->sc_ioh,
313				sc->sc_config_offset + index);
314}
315
316uint64_t
317virtio_read_device_config_8(struct virtio_softc *sc, int index)
318{
319	uint64_t r;
320
321	r = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
322			     sc->sc_config_offset + index + sizeof(uint32_t));
323	r <<= 32;
324	r += bus_space_read_4(sc->sc_iot, sc->sc_ioh,
325			      sc->sc_config_offset + index);
326	return r;
327}
328
329void
330virtio_write_device_config_1(struct virtio_softc *sc,
331			     int index, uint8_t value)
332{
333	bus_space_write_1(sc->sc_iot, sc->sc_ioh,
334			  sc->sc_config_offset + index, value);
335}
336
337void
338virtio_write_device_config_2(struct virtio_softc *sc,
339			     int index, uint16_t value)
340{
341	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
342			  sc->sc_config_offset + index, value);
343}
344
345void
346virtio_write_device_config_4(struct virtio_softc *sc,
347			     int index, uint32_t value)
348{
349	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
350			  sc->sc_config_offset + index, value);
351}
352
353void
354virtio_write_device_config_8(struct virtio_softc *sc,
355			     int index, uint64_t value)
356{
357	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
358			  sc->sc_config_offset + index,
359			  value & 0xffffffff);
360	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
361			  sc->sc_config_offset + index + sizeof(uint32_t),
362			  value >> 32);
363}
364
365/*
366 * Interrupt handler.
367 */
368static int
369virtio_intr(void *arg)
370{
371	struct virtio_softc *sc = arg;
372	int isr, r = 0;
373
374	/* check and ack the interrupt */
375	isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
376			       VIRTIO_CONFIG_ISR_STATUS);
377	if (isr == 0)
378		return 0;
379	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
380	    (sc->sc_config_change != NULL))
381		r = (sc->sc_config_change)(sc);
382	if (sc->sc_intrhand != NULL)
383		r |= (sc->sc_intrhand)(sc);
384
385	return r;
386}
387
388/*
389 * dmamap sync operations for a virtqueue.
390 */
391static inline void
392vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
393{
394	/* availoffset == sizeof(vring_desc)*vq_num */
395	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
396			ops);
397}
398
399static inline void
400vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
401{
402	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
403			vq->vq_availoffset,
404			offsetof(struct vring_avail, ring)
405			 + vq->vq_num * sizeof(uint16_t),
406			ops);
407}
408
409static inline void
410vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
411{
412	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
413			vq->vq_usedoffset,
414			offsetof(struct vring_used, ring)
415			 + vq->vq_num * sizeof(struct vring_used_elem),
416			ops);
417}
418
419static inline void
420vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
421		     int ops)
422{
423	int offset = vq->vq_indirectoffset
424		      + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
425
426	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
427			offset, sizeof(struct vring_desc) * vq->vq_maxnsegs,
428			ops);
429}
430
431/*
432 * Can be used as sc_intrhand.
433 */
434/*
435 * Scan vq, bus_dmamap_sync for the vqs (not for the payload),
436 * and calls (*vq_done)() if some entries are consumed.
437 */
438int
439virtio_vq_intr(struct virtio_softc *sc)
440{
441	struct virtqueue *vq;
442	int i, r = 0;
443
444	for (i = 0; i < sc->sc_nvqs; i++) {
445		vq = &sc->sc_vqs[i];
446		if (vq->vq_queued) {
447			vq->vq_queued = 0;
448			vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
449		}
450		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
451		membar_consumer();
452		if (vq->vq_used_idx != vq->vq_used->idx) {
453			if (vq->vq_done)
454				r |= (vq->vq_done)(vq);
455		}
456	}
457
458
459	return r;
460}
461
462/*
463 * Start/stop vq interrupt.  No guarantee.
464 */
465void
466virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
467{
468	vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
469	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
470	vq->vq_queued++;
471}
472
473void
474virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
475{
476	vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
477	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
478	vq->vq_queued++;
479}
480
481/*
482 * Initialize vq structure.
483 */
484static void
485virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit)
486{
487	int i, j;
488	int vq_size = vq->vq_num;
489
490	memset(vq->vq_vaddr, 0, vq->vq_bytesize);
491
492	/* build the indirect descriptor chain */
493	if (vq->vq_indirect != NULL) {
494		struct vring_desc *vd;
495
496		for (i = 0; i < vq_size; i++) {
497			vd = vq->vq_indirect;
498			vd += vq->vq_maxnsegs * i;
499			for (j = 0; j < vq->vq_maxnsegs-1; j++)
500				vd[j].next = j + 1;
501		}
502	}
503
504	/* free slot management */
505	SIMPLEQ_INIT(&vq->vq_freelist);
506	for (i = 0; i < vq_size; i++) {
507		SIMPLEQ_INSERT_TAIL(&vq->vq_freelist,
508				    &vq->vq_entries[i], qe_list);
509		vq->vq_entries[i].qe_index = i;
510	}
511	if (!reinit)
512		mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl);
513
514	/* enqueue/dequeue status */
515	vq->vq_avail_idx = 0;
516	vq->vq_used_idx = 0;
517	vq->vq_queued = 0;
518	if (!reinit) {
519		mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
520		mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
521	}
522	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
523	vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
524	vq->vq_queued++;
525}
526
527/*
528 * Allocate/free a vq.
529 */
530int
531virtio_alloc_vq(struct virtio_softc *sc,
532		struct virtqueue *vq, int index, int maxsegsize, int maxnsegs,
533		const char *name)
534{
535	int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0;
536	int rsegs, r;
537#define VIRTQUEUE_ALIGN(n)	(((n)+(VIRTIO_PAGE_SIZE-1))&	\
538				 ~(VIRTIO_PAGE_SIZE-1))
539
540	memset(vq, 0, sizeof(*vq));
541
542	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
543			  VIRTIO_CONFIG_QUEUE_SELECT, index);
544	vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh,
545				   VIRTIO_CONFIG_QUEUE_SIZE);
546	if (vq_size == 0) {
547		aprint_error_dev(sc->sc_dev,
548				 "virtqueue not exist, index %d for %s\n",
549				 index, name);
550		goto err;
551	}
552	/* allocsize1: descriptor table + avail ring + pad */
553	allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size
554				     + sizeof(uint16_t)*(2+vq_size));
555	/* allocsize2: used ring + pad */
556	allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2
557				     + sizeof(struct vring_used_elem)*vq_size);
558	/* allocsize3: indirect table */
559	if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT)
560		allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size;
561	else
562		allocsize3 = 0;
563	allocsize = allocsize1 + allocsize2 + allocsize3;
564
565	/* alloc and map the memory */
566	r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
567			     &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
568	if (r != 0) {
569		aprint_error_dev(sc->sc_dev,
570				 "virtqueue %d for %s allocation failed, "
571				 "error code %d\n", index, name, r);
572		goto err;
573	}
574	r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize,
575			   &vq->vq_vaddr, BUS_DMA_NOWAIT);
576	if (r != 0) {
577		aprint_error_dev(sc->sc_dev,
578				 "virtqueue %d for %s map failed, "
579				 "error code %d\n", index, name, r);
580		goto err;
581	}
582	r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
583			      BUS_DMA_NOWAIT, &vq->vq_dmamap);
584	if (r != 0) {
585		aprint_error_dev(sc->sc_dev,
586				 "virtqueue %d for %s dmamap creation failed, "
587				 "error code %d\n", index, name, r);
588		goto err;
589	}
590	r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
591			    vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT);
592	if (r != 0) {
593		aprint_error_dev(sc->sc_dev,
594				 "virtqueue %d for %s dmamap load failed, "
595				 "error code %d\n", index, name, r);
596		goto err;
597	}
598
599	/* set the vq address */
600	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
601			  VIRTIO_CONFIG_QUEUE_ADDRESS,
602			  (vq->vq_dmamap->dm_segs[0].ds_addr
603			   / VIRTIO_PAGE_SIZE));
604
605	/* remember addresses and offsets for later use */
606	vq->vq_owner = sc;
607	vq->vq_num = vq_size;
608	vq->vq_index = index;
609	vq->vq_desc = vq->vq_vaddr;
610	vq->vq_availoffset = sizeof(struct vring_desc)*vq_size;
611	vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset);
612	vq->vq_usedoffset = allocsize1;
613	vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset);
614	if (allocsize3 > 0) {
615		vq->vq_indirectoffset = allocsize1 + allocsize2;
616		vq->vq_indirect = (void*)(((char*)vq->vq_desc)
617					  + vq->vq_indirectoffset);
618	}
619	vq->vq_bytesize = allocsize;
620	vq->vq_maxsegsize = maxsegsize;
621	vq->vq_maxnsegs = maxnsegs;
622
623	/* free slot management */
624	vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size,
625				     KM_NOSLEEP);
626	if (vq->vq_entries == NULL) {
627		r = ENOMEM;
628		goto err;
629	}
630
631	virtio_init_vq(sc, vq, false);
632
633	aprint_verbose_dev(sc->sc_dev,
634			   "allocated %u byte for virtqueue %d for %s, "
635			   "size %d\n", allocsize, index, name, vq_size);
636	if (allocsize3 > 0)
637		aprint_verbose_dev(sc->sc_dev,
638				   "using %d byte (%d entries) "
639				   "indirect descriptors\n",
640				   allocsize3, maxnsegs * vq_size);
641	return 0;
642
643err:
644	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
645			  VIRTIO_CONFIG_QUEUE_ADDRESS, 0);
646	if (vq->vq_dmamap)
647		bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
648	if (vq->vq_vaddr)
649		bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
650	if (vq->vq_segs[0].ds_addr)
651		bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
652	memset(vq, 0, sizeof(*vq));
653
654	return -1;
655}
656
657int
658virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
659{
660	struct vq_entry *qe;
661	int i = 0;
662
663	/* device must be already deactivated */
664	/* confirm the vq is empty */
665	SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) {
666		i++;
667	}
668	if (i != vq->vq_num) {
669		printf("%s: freeing non-empty vq, index %d\n",
670		       device_xname(sc->sc_dev), vq->vq_index);
671		return EBUSY;
672	}
673
674	/* tell device that there's no virtqueue any longer */
675	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
676			  VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index);
677	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
678			  VIRTIO_CONFIG_QUEUE_ADDRESS, 0);
679
680	kmem_free(vq->vq_entries, vq->vq_bytesize);
681	bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
682	bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
683	bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
684	bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
685	mutex_destroy(&vq->vq_freelist_lock);
686	mutex_destroy(&vq->vq_uring_lock);
687	mutex_destroy(&vq->vq_aring_lock);
688	memset(vq, 0, sizeof(*vq));
689
690	return 0;
691}
692
693/*
694 * Free descriptor management.
695 */
696static struct vq_entry *
697vq_alloc_entry(struct virtqueue *vq)
698{
699	struct vq_entry *qe;
700
701	mutex_enter(&vq->vq_freelist_lock);
702	if (SIMPLEQ_EMPTY(&vq->vq_freelist)) {
703		mutex_exit(&vq->vq_freelist_lock);
704		return NULL;
705	}
706	qe = SIMPLEQ_FIRST(&vq->vq_freelist);
707	SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list);
708	mutex_exit(&vq->vq_freelist_lock);
709
710	return qe;
711}
712
713static void
714vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
715{
716	mutex_enter(&vq->vq_freelist_lock);
717	SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list);
718	mutex_exit(&vq->vq_freelist_lock);
719
720	return;
721}
722
723/*
724 * Enqueue several dmamaps as a single request.
725 */
726/*
727 * Typical usage:
728 *  <queue size> number of followings are stored in arrays
729 *  - command blocks (in dmamem) should be pre-allocated and mapped
730 *  - dmamaps for command blocks should be pre-allocated and loaded
731 *  - dmamaps for payload should be pre-allocated
732 *      r = virtio_enqueue_prep(sc, vq, &slot);		// allocate a slot
733 *	if (r)		// currently 0 or EAGAIN
734 *	  return r;
735 *	r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
736 *	if (r) {
737 *	  virtio_enqueue_abort(sc, vq, slot);
738 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
739 *	  return r;
740 *	}
741 *	r = virtio_enqueue_reserve(sc, vq, slot,
742 *				   dmamap_payload[slot]->dm_nsegs+1);
743 *							// ^ +1 for command
744 *	if (r) {	// currently 0 or EAGAIN
745 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
746 *	  return r;					// do not call abort()
747 *	}
748 *	<setup and prepare commands>
749 *	bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
750 *	bus_dmamap_sync(dmat, dmamap_payload[slot],...);
751 *	virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
752 *	virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
753 *	virtio_enqueue_commit(sc, vq, slot, true);
754 */
755
756/*
757 * enqueue_prep: allocate a slot number
758 */
759int
760virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
761{
762	struct vq_entry *qe1;
763
764	KASSERT(slotp != NULL);
765
766	qe1 = vq_alloc_entry(vq);
767	if (qe1 == NULL)
768		return EAGAIN;
769	/* next slot is not allocated yet */
770	qe1->qe_next = -1;
771	*slotp = qe1->qe_index;
772
773	return 0;
774}
775
776/*
777 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
778 */
779int
780virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
781		       int slot, int nsegs)
782{
783	int indirect;
784	struct vq_entry *qe1 = &vq->vq_entries[slot];
785
786	KASSERT(qe1->qe_next == -1);
787	KASSERT(1 <= nsegs && nsegs <= vq->vq_num);
788
789	if ((vq->vq_indirect != NULL) &&
790	    (nsegs >= MINSEG_INDIRECT) &&
791	    (nsegs <= vq->vq_maxnsegs))
792		indirect = 1;
793	else
794		indirect = 0;
795	qe1->qe_indirect = indirect;
796
797	if (indirect) {
798		struct vring_desc *vd;
799		int i;
800
801		vd = &vq->vq_desc[qe1->qe_index];
802		vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr
803			+ vq->vq_indirectoffset;
804		vd->addr += sizeof(struct vring_desc)
805			* vq->vq_maxnsegs * qe1->qe_index;
806		vd->len = sizeof(struct vring_desc) * nsegs;
807		vd->flags = VRING_DESC_F_INDIRECT;
808
809		vd = vq->vq_indirect;
810		vd += vq->vq_maxnsegs * qe1->qe_index;
811		qe1->qe_desc_base = vd;
812
813		for (i = 0; i < nsegs-1; i++) {
814			vd[i].flags = VRING_DESC_F_NEXT;
815		}
816		vd[i].flags = 0;
817		qe1->qe_next = 0;
818
819		return 0;
820	} else {
821		struct vring_desc *vd;
822		struct vq_entry *qe;
823		int i, s;
824
825		vd = &vq->vq_desc[0];
826		qe1->qe_desc_base = vd;
827		qe1->qe_next = qe1->qe_index;
828		s = slot;
829		for (i = 0; i < nsegs - 1; i++) {
830			qe = vq_alloc_entry(vq);
831			if (qe == NULL) {
832				vd[s].flags = 0;
833				virtio_enqueue_abort(sc, vq, slot);
834				return EAGAIN;
835			}
836			vd[s].flags = VRING_DESC_F_NEXT;
837			vd[s].next = qe->qe_index;
838			s = qe->qe_index;
839		}
840		vd[s].flags = 0;
841
842		return 0;
843	}
844}
845
846/*
847 * enqueue: enqueue a single dmamap.
848 */
849int
850virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
851	       bus_dmamap_t dmamap, bool write)
852{
853	struct vq_entry *qe1 = &vq->vq_entries[slot];
854	struct vring_desc *vd = qe1->qe_desc_base;
855	int i;
856	int s = qe1->qe_next;
857
858	KASSERT(s >= 0);
859	KASSERT(dmamap->dm_nsegs > 0);
860
861	for (i = 0; i < dmamap->dm_nsegs; i++) {
862		vd[s].addr = dmamap->dm_segs[i].ds_addr;
863		vd[s].len = dmamap->dm_segs[i].ds_len;
864		if (!write)
865			vd[s].flags |= VRING_DESC_F_WRITE;
866		s = vd[s].next;
867	}
868	qe1->qe_next = s;
869
870	return 0;
871}
872
873int
874virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
875		 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
876		 bool write)
877{
878	struct vq_entry *qe1 = &vq->vq_entries[slot];
879	struct vring_desc *vd = qe1->qe_desc_base;
880	int s = qe1->qe_next;
881
882	KASSERT(s >= 0);
883	KASSERT(dmamap->dm_nsegs == 1); /* XXX */
884	KASSERT((dmamap->dm_segs[0].ds_len > start) &&
885		(dmamap->dm_segs[0].ds_len >= start + len));
886
887	vd[s].addr = dmamap->dm_segs[0].ds_addr + start;
888	vd[s].len = len;
889	if (!write)
890		vd[s].flags |= VRING_DESC_F_WRITE;
891	qe1->qe_next = vd[s].next;
892
893	return 0;
894}
895
896/*
897 * enqueue_commit: add it to the aring.
898 */
899int
900virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
901		      bool notifynow)
902{
903	struct vq_entry *qe1;
904
905	if (slot < 0) {
906		mutex_enter(&vq->vq_aring_lock);
907		goto notify;
908	}
909	vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
910	qe1 = &vq->vq_entries[slot];
911	if (qe1->qe_indirect)
912		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
913	mutex_enter(&vq->vq_aring_lock);
914	vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot;
915
916notify:
917	if (notifynow) {
918		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
919		vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
920		membar_producer();
921		vq->vq_avail->idx = vq->vq_avail_idx;
922		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
923		membar_producer();
924		vq->vq_queued++;
925		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
926		membar_consumer();
927		if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY))
928			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
929					  VIRTIO_CONFIG_QUEUE_NOTIFY,
930					  vq->vq_index);
931	}
932	mutex_exit(&vq->vq_aring_lock);
933
934	return 0;
935}
936
937/*
938 * enqueue_abort: rollback.
939 */
940int
941virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
942{
943	struct vq_entry *qe = &vq->vq_entries[slot];
944	struct vring_desc *vd;
945	int s;
946
947	if (qe->qe_next < 0) {
948		vq_free_entry(vq, qe);
949		return 0;
950	}
951
952	s = slot;
953	vd = &vq->vq_desc[0];
954	while (vd[s].flags & VRING_DESC_F_NEXT) {
955		s = vd[s].next;
956		vq_free_entry(vq, qe);
957		qe = &vq->vq_entries[s];
958	}
959	vq_free_entry(vq, qe);
960	return 0;
961}
962
963/*
964 * Dequeue a request.
965 */
966/*
967 * dequeue: dequeue a request from uring; dmamap_sync for uring is
968 *	    already done in the interrupt handler.
969 */
970int
971virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
972	       int *slotp, int *lenp)
973{
974	uint16_t slot, usedidx;
975	struct vq_entry *qe;
976
977	if (vq->vq_used_idx == vq->vq_used->idx)
978		return ENOENT;
979	mutex_enter(&vq->vq_uring_lock);
980	usedidx = vq->vq_used_idx++;
981	mutex_exit(&vq->vq_uring_lock);
982	usedidx %= vq->vq_num;
983	slot = vq->vq_used->ring[usedidx].id;
984	qe = &vq->vq_entries[slot];
985
986	if (qe->qe_indirect)
987		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
988
989	if (slotp)
990		*slotp = slot;
991	if (lenp)
992		*lenp = vq->vq_used->ring[usedidx].len;
993
994	return 0;
995}
996
997/*
998 * dequeue_commit: complete dequeue; the slot is recycled for future use.
999 *                 if you forget to call this the slot will be leaked.
1000 */
1001int
1002virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
1003{
1004	struct vq_entry *qe = &vq->vq_entries[slot];
1005	struct vring_desc *vd = &vq->vq_desc[0];
1006	int s = slot;
1007
1008	while (vd[s].flags & VRING_DESC_F_NEXT) {
1009		s = vd[s].next;
1010		vq_free_entry(vq, qe);
1011		qe = &vq->vq_entries[s];
1012	}
1013	vq_free_entry(vq, qe);
1014
1015	return 0;
1016}
1017