virtio.c revision 1.7 
1/*	$NetBSD: virtio.c,v 1.7 2014/10/06 13:31:54 mlelstv Exp $	*/
2
3/*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.7 2014/10/06 13:31:54 mlelstv Exp $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/atomic.h>
35#include <sys/bus.h>
36#include <sys/device.h>
37#include <sys/kmem.h>
38
39#include <dev/pci/pcidevs.h>
40#include <dev/pci/pcireg.h>
41#include <dev/pci/pcivar.h>
42
43#include <dev/pci/virtioreg.h>
44#include <dev/pci/virtiovar.h>
45
46#define MINSEG_INDIRECT		2 /* use indirect if nsegs >= this value */
47
48static int	virtio_match(device_t, cfdata_t, void *);
49static void	virtio_attach(device_t, device_t, void *);
50static int	virtio_detach(device_t, int);
51static int	virtio_intr(void *arg);
52static void	virtio_init_vq(struct virtio_softc *,
53		    struct virtqueue *, const bool);
54
55CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc),
56    virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL,
57    DVF_DETACH_SHUTDOWN);
58
59static void
60virtio_set_status(struct virtio_softc *sc, int status)
61{
62	int old = 0;
63
64	if (status != 0)
65		old = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
66				       VIRTIO_CONFIG_DEVICE_STATUS);
67	bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS,
68			  status|old);
69}
70
71#define virtio_device_reset(sc)	virtio_set_status((sc), 0)
72
73static int
74virtio_match(device_t parent, cfdata_t match, void *aux)
75{
76	struct pci_attach_args *pa;
77
78	pa = (struct pci_attach_args *)aux;
79	switch (PCI_VENDOR(pa->pa_id)) {
80	case PCI_VENDOR_QUMRANET:
81		if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
82		     PCI_PRODUCT(pa->pa_id)) &&
83		    (PCI_PRODUCT(pa->pa_id) <=
84		     PCI_PRODUCT_QUMRANET_VIRTIO_103F))
85			return 1;
86		break;
87	}
88
89	return 0;
90}
91
92static const char *virtio_device_name[] = {
93	"Unknown (0)",		/* 0 */
94	"Network",		/* 1 */
95	"Block",		/* 2 */
96	"Console",		/* 3 */
97	"Entropy",		/* 4 */
98	"Memory Balloon",	/* 5 */
99	"Unknown (6)",		/* 6 */
100	"Unknown (7)",		/* 7 */
101	"Unknown (8)",		/* 8 */
102	"9P Transport"		/* 9 */
103};
104#define NDEVNAMES	(sizeof(virtio_device_name)/sizeof(char*))
105
106static void
107virtio_attach(device_t parent, device_t self, void *aux)
108{
109	struct virtio_softc *sc = device_private(self);
110	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
111	pci_chipset_tag_t pc = pa->pa_pc;
112	pcitag_t tag = pa->pa_tag;
113	int revision;
114	pcireg_t id;
115	char const *intrstr;
116	pci_intr_handle_t ih;
117	char intrbuf[PCI_INTRSTR_LEN];
118
119	revision = PCI_REVISION(pa->pa_class);
120	if (revision != 0) {
121		aprint_normal(": unknown revision 0x%02x; giving up\n",
122			      revision);
123		return;
124	}
125	aprint_normal("\n");
126	aprint_naive("\n");
127
128	/* subsystem ID shows what I am */
129	id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
130	aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
131			  (PCI_SUBSYS_ID(id) < NDEVNAMES?
132			   virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"),
133			  revision);
134
135	sc->sc_dev = self;
136	sc->sc_pc = pc;
137	sc->sc_tag = tag;
138	sc->sc_iot = pa->pa_iot;
139	sc->sc_dmat = pa->pa_dmat;
140	sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
141
142	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
143			   &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) {
144		aprint_error_dev(self, "can't map i/o space\n");
145		return;
146	}
147
148	virtio_device_reset(sc);
149	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
150	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
151
152	/* XXX: use softc as aux... */
153	sc->sc_childdevid = PCI_SUBSYS_ID(id);
154	sc->sc_child = NULL;
155	config_found(self, sc, NULL);
156	if (sc->sc_child == NULL) {
157		aprint_error_dev(self,
158				 "no matching child driver; not configured\n");
159		return;
160	}
161	if (sc->sc_child == (void*)1) { /* this shows error */
162		aprint_error_dev(self,
163				 "virtio configuration failed\n");
164		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
165		return;
166	}
167
168	if (pci_intr_map(pa, &ih)) {
169		aprint_error_dev(self, "couldn't map interrupt\n");
170		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
171		return;
172	}
173
174	intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf));
175
176	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
177		pci_intr_setattr(pc, &ih, PCI_INTR_MPSAFE, true);
178
179	sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc);
180
181	if (sc->sc_ih == NULL) {
182		aprint_error_dev(self, "couldn't establish interrupt");
183		if (intrstr != NULL)
184			aprint_error(" at %s", intrstr);
185		aprint_error("\n");
186		virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
187		return;
188	}
189	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
190
191	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
192
193	return;
194}
195
196static int
197virtio_detach(device_t self, int flags)
198{
199	struct virtio_softc *sc = device_private(self);
200	int r;
201
202	if (sc->sc_child != 0 && sc->sc_child != (void*)1) {
203		r = config_detach(sc->sc_child, flags);
204		if (r)
205			return r;
206	}
207	KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1);
208	KASSERT(sc->sc_vqs == 0);
209	if (sc->sc_ih != NULL) {
210		pci_intr_disestablish(sc->sc_pc, sc->sc_ih);
211		sc->sc_ih = NULL;
212	}
213	if (sc->sc_iosize)
214		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
215	sc->sc_iosize = 0;
216
217	return 0;
218}
219
220/*
221 * Reset the device.
222 */
223/*
224 * To reset the device to a known state, do following:
225 *	virtio_reset(sc);	     // this will stop the device activity
226 *	<dequeue finished requests>; // virtio_dequeue() still can be called
227 *	<revoke pending requests in the vqs if any>;
228 *	virtio_reinit_begin(sc);     // dequeue prohibitted
229 *	newfeatures = virtio_negotiate_features(sc, requestedfeatures);
230 *	<some other initialization>;
231 *	virtio_reinit_end(sc);	     // device activated; enqueue allowed
232 * Once attached, feature negotiation can only be allowed after virtio_reset.
233 */
234void
235virtio_reset(struct virtio_softc *sc)
236{
237	virtio_device_reset(sc);
238}
239
240void
241virtio_reinit_start(struct virtio_softc *sc)
242{
243	int i;
244
245	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
246	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
247	for (i = 0; i < sc->sc_nvqs; i++) {
248		int n;
249		struct virtqueue *vq = &sc->sc_vqs[i];
250		bus_space_write_2(sc->sc_iot, sc->sc_ioh,
251				  VIRTIO_CONFIG_QUEUE_SELECT,
252				  vq->vq_index);
253		n = bus_space_read_2(sc->sc_iot, sc->sc_ioh,
254				     VIRTIO_CONFIG_QUEUE_SIZE);
255		if (n == 0)	/* vq disappeared */
256			continue;
257		if (n != vq->vq_num) {
258			panic("%s: virtqueue size changed, vq index %d\n",
259			      device_xname(sc->sc_dev),
260			      vq->vq_index);
261		}
262		virtio_init_vq(sc, vq, true);
263		bus_space_write_4(sc->sc_iot, sc->sc_ioh,
264				  VIRTIO_CONFIG_QUEUE_ADDRESS,
265				  (vq->vq_dmamap->dm_segs[0].ds_addr
266				   / VIRTIO_PAGE_SIZE));
267	}
268}
269
270void
271virtio_reinit_end(struct virtio_softc *sc)
272{
273	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
274}
275
276/*
277 * Feature negotiation.
278 */
279uint32_t
280virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
281{
282	uint32_t r;
283
284	if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
285	    !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
286		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
287	r = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
288			     VIRTIO_CONFIG_DEVICE_FEATURES);
289	r &= guest_features;
290	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
291			  VIRTIO_CONFIG_GUEST_FEATURES, r);
292	sc->sc_features = r;
293	if (r & VIRTIO_F_RING_INDIRECT_DESC)
294		sc->sc_indirect = true;
295	else
296		sc->sc_indirect = false;
297
298	return r;
299}
300
301/*
302 * Device configuration registers.
303 */
304uint8_t
305virtio_read_device_config_1(struct virtio_softc *sc, int index)
306{
307	return bus_space_read_1(sc->sc_iot, sc->sc_ioh,
308				sc->sc_config_offset + index);
309}
310
311uint16_t
312virtio_read_device_config_2(struct virtio_softc *sc, int index)
313{
314	return bus_space_read_2(sc->sc_iot, sc->sc_ioh,
315				sc->sc_config_offset + index);
316}
317
318uint32_t
319virtio_read_device_config_4(struct virtio_softc *sc, int index)
320{
321	return bus_space_read_4(sc->sc_iot, sc->sc_ioh,
322				sc->sc_config_offset + index);
323}
324
325uint64_t
326virtio_read_device_config_8(struct virtio_softc *sc, int index)
327{
328	uint64_t r;
329
330	r = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
331			     sc->sc_config_offset + index + sizeof(uint32_t));
332	r <<= 32;
333	r += bus_space_read_4(sc->sc_iot, sc->sc_ioh,
334			      sc->sc_config_offset + index);
335	return r;
336}
337
338void
339virtio_write_device_config_1(struct virtio_softc *sc,
340			     int index, uint8_t value)
341{
342	bus_space_write_1(sc->sc_iot, sc->sc_ioh,
343			  sc->sc_config_offset + index, value);
344}
345
346void
347virtio_write_device_config_2(struct virtio_softc *sc,
348			     int index, uint16_t value)
349{
350	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
351			  sc->sc_config_offset + index, value);
352}
353
354void
355virtio_write_device_config_4(struct virtio_softc *sc,
356			     int index, uint32_t value)
357{
358	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
359			  sc->sc_config_offset + index, value);
360}
361
362void
363virtio_write_device_config_8(struct virtio_softc *sc,
364			     int index, uint64_t value)
365{
366	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
367			  sc->sc_config_offset + index,
368			  value & 0xffffffff);
369	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
370			  sc->sc_config_offset + index + sizeof(uint32_t),
371			  value >> 32);
372}
373
374/*
375 * Interrupt handler.
376 */
377static int
378virtio_intr(void *arg)
379{
380	struct virtio_softc *sc = arg;
381	int isr, r = 0;
382
383	/* check and ack the interrupt */
384	isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
385			       VIRTIO_CONFIG_ISR_STATUS);
386	if (isr == 0)
387		return 0;
388	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
389	    (sc->sc_config_change != NULL))
390		r = (sc->sc_config_change)(sc);
391	if (sc->sc_intrhand != NULL)
392		r |= (sc->sc_intrhand)(sc);
393
394	return r;
395}
396
397/*
398 * dmamap sync operations for a virtqueue.
399 */
400static inline void
401vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
402{
403	/* availoffset == sizeof(vring_desc)*vq_num */
404	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
405			ops);
406}
407
408static inline void
409vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
410{
411	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
412			vq->vq_availoffset,
413			offsetof(struct vring_avail, ring)
414			 + vq->vq_num * sizeof(uint16_t),
415			ops);
416}
417
418static inline void
419vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
420{
421	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
422			vq->vq_usedoffset,
423			offsetof(struct vring_used, ring)
424			 + vq->vq_num * sizeof(struct vring_used_elem),
425			ops);
426}
427
428static inline void
429vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
430		     int ops)
431{
432	int offset = vq->vq_indirectoffset
433		      + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
434
435	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
436			offset, sizeof(struct vring_desc) * vq->vq_maxnsegs,
437			ops);
438}
439
440/*
441 * Can be used as sc_intrhand.
442 */
443/*
444 * Scan vq, bus_dmamap_sync for the vqs (not for the payload),
445 * and calls (*vq_done)() if some entries are consumed.
446 */
447int
448virtio_vq_intr(struct virtio_softc *sc)
449{
450	struct virtqueue *vq;
451	int i, r = 0;
452
453	for (i = 0; i < sc->sc_nvqs; i++) {
454		vq = &sc->sc_vqs[i];
455		if (vq->vq_queued) {
456			vq->vq_queued = 0;
457			vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
458		}
459		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
460		membar_consumer();
461		if (vq->vq_used_idx != vq->vq_used->idx) {
462			if (vq->vq_done)
463				r |= (vq->vq_done)(vq);
464		}
465	}
466
467
468	return r;
469}
470
471/*
472 * Start/stop vq interrupt.  No guarantee.
473 */
474void
475virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
476{
477	vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
478	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
479	vq->vq_queued++;
480}
481
482void
483virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
484{
485	vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
486	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
487	vq->vq_queued++;
488}
489
490/*
491 * Initialize vq structure.
492 */
493static void
494virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit)
495{
496	int i, j;
497	int vq_size = vq->vq_num;
498
499	memset(vq->vq_vaddr, 0, vq->vq_bytesize);
500
501	/* build the indirect descriptor chain */
502	if (vq->vq_indirect != NULL) {
503		struct vring_desc *vd;
504
505		for (i = 0; i < vq_size; i++) {
506			vd = vq->vq_indirect;
507			vd += vq->vq_maxnsegs * i;
508			for (j = 0; j < vq->vq_maxnsegs-1; j++)
509				vd[j].next = j + 1;
510		}
511	}
512
513	/* free slot management */
514	SIMPLEQ_INIT(&vq->vq_freelist);
515	for (i = 0; i < vq_size; i++) {
516		SIMPLEQ_INSERT_TAIL(&vq->vq_freelist,
517				    &vq->vq_entries[i], qe_list);
518		vq->vq_entries[i].qe_index = i;
519	}
520	if (!reinit)
521		mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl);
522
523	/* enqueue/dequeue status */
524	vq->vq_avail_idx = 0;
525	vq->vq_used_idx = 0;
526	vq->vq_queued = 0;
527	if (!reinit) {
528		mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
529		mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
530	}
531	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
532	vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
533	vq->vq_queued++;
534}
535
536/*
537 * Allocate/free a vq.
538 */
539int
540virtio_alloc_vq(struct virtio_softc *sc,
541		struct virtqueue *vq, int index, int maxsegsize, int maxnsegs,
542		const char *name)
543{
544	int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0;
545	int rsegs, r;
546#define VIRTQUEUE_ALIGN(n)	(((n)+(VIRTIO_PAGE_SIZE-1))&	\
547				 ~(VIRTIO_PAGE_SIZE-1))
548
549	memset(vq, 0, sizeof(*vq));
550
551	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
552			  VIRTIO_CONFIG_QUEUE_SELECT, index);
553	vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh,
554				   VIRTIO_CONFIG_QUEUE_SIZE);
555	if (vq_size == 0) {
556		aprint_error_dev(sc->sc_dev,
557				 "virtqueue not exist, index %d for %s\n",
558				 index, name);
559		goto err;
560	}
561	/* allocsize1: descriptor table + avail ring + pad */
562	allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size
563				     + sizeof(uint16_t)*(2+vq_size));
564	/* allocsize2: used ring + pad */
565	allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2
566				     + sizeof(struct vring_used_elem)*vq_size);
567	/* allocsize3: indirect table */
568	if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT)
569		allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size;
570	else
571		allocsize3 = 0;
572	allocsize = allocsize1 + allocsize2 + allocsize3;
573
574	/* alloc and map the memory */
575	r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
576			     &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
577	if (r != 0) {
578		aprint_error_dev(sc->sc_dev,
579				 "virtqueue %d for %s allocation failed, "
580				 "error code %d\n", index, name, r);
581		goto err;
582	}
583	r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize,
584			   &vq->vq_vaddr, BUS_DMA_NOWAIT);
585	if (r != 0) {
586		aprint_error_dev(sc->sc_dev,
587				 "virtqueue %d for %s map failed, "
588				 "error code %d\n", index, name, r);
589		goto err;
590	}
591	r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
592			      BUS_DMA_NOWAIT, &vq->vq_dmamap);
593	if (r != 0) {
594		aprint_error_dev(sc->sc_dev,
595				 "virtqueue %d for %s dmamap creation failed, "
596				 "error code %d\n", index, name, r);
597		goto err;
598	}
599	r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
600			    vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT);
601	if (r != 0) {
602		aprint_error_dev(sc->sc_dev,
603				 "virtqueue %d for %s dmamap load failed, "
604				 "error code %d\n", index, name, r);
605		goto err;
606	}
607
608	/* set the vq address */
609	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
610			  VIRTIO_CONFIG_QUEUE_ADDRESS,
611			  (vq->vq_dmamap->dm_segs[0].ds_addr
612			   / VIRTIO_PAGE_SIZE));
613
614	/* remember addresses and offsets for later use */
615	vq->vq_owner = sc;
616	vq->vq_num = vq_size;
617	vq->vq_index = index;
618	vq->vq_desc = vq->vq_vaddr;
619	vq->vq_availoffset = sizeof(struct vring_desc)*vq_size;
620	vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset);
621	vq->vq_usedoffset = allocsize1;
622	vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset);
623	if (allocsize3 > 0) {
624		vq->vq_indirectoffset = allocsize1 + allocsize2;
625		vq->vq_indirect = (void*)(((char*)vq->vq_desc)
626					  + vq->vq_indirectoffset);
627	}
628	vq->vq_bytesize = allocsize;
629	vq->vq_maxsegsize = maxsegsize;
630	vq->vq_maxnsegs = maxnsegs;
631
632	/* free slot management */
633	vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size,
634				     KM_NOSLEEP);
635	if (vq->vq_entries == NULL) {
636		r = ENOMEM;
637		goto err;
638	}
639
640	virtio_init_vq(sc, vq, false);
641
642	aprint_verbose_dev(sc->sc_dev,
643			   "allocated %u byte for virtqueue %d for %s, "
644			   "size %d\n", allocsize, index, name, vq_size);
645	if (allocsize3 > 0)
646		aprint_verbose_dev(sc->sc_dev,
647				   "using %d byte (%d entries) "
648				   "indirect descriptors\n",
649				   allocsize3, maxnsegs * vq_size);
650	return 0;
651
652err:
653	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
654			  VIRTIO_CONFIG_QUEUE_ADDRESS, 0);
655	if (vq->vq_dmamap)
656		bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
657	if (vq->vq_vaddr)
658		bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
659	if (vq->vq_segs[0].ds_addr)
660		bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
661	memset(vq, 0, sizeof(*vq));
662
663	return -1;
664}
665
666int
667virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
668{
669	struct vq_entry *qe;
670	int i = 0;
671
672	/* device must be already deactivated */
673	/* confirm the vq is empty */
674	SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) {
675		i++;
676	}
677	if (i != vq->vq_num) {
678		printf("%s: freeing non-empty vq, index %d\n",
679		       device_xname(sc->sc_dev), vq->vq_index);
680		return EBUSY;
681	}
682
683	/* tell device that there's no virtqueue any longer */
684	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
685			  VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index);
686	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
687			  VIRTIO_CONFIG_QUEUE_ADDRESS, 0);
688
689	kmem_free(vq->vq_entries, vq->vq_bytesize);
690	bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
691	bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
692	bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
693	bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
694	mutex_destroy(&vq->vq_freelist_lock);
695	mutex_destroy(&vq->vq_uring_lock);
696	mutex_destroy(&vq->vq_aring_lock);
697	memset(vq, 0, sizeof(*vq));
698
699	return 0;
700}
701
702/*
703 * Free descriptor management.
704 */
705static struct vq_entry *
706vq_alloc_entry(struct virtqueue *vq)
707{
708	struct vq_entry *qe;
709
710	mutex_enter(&vq->vq_freelist_lock);
711	if (SIMPLEQ_EMPTY(&vq->vq_freelist)) {
712		mutex_exit(&vq->vq_freelist_lock);
713		return NULL;
714	}
715	qe = SIMPLEQ_FIRST(&vq->vq_freelist);
716	SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list);
717	mutex_exit(&vq->vq_freelist_lock);
718
719	return qe;
720}
721
722static void
723vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
724{
725	mutex_enter(&vq->vq_freelist_lock);
726	SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list);
727	mutex_exit(&vq->vq_freelist_lock);
728
729	return;
730}
731
732/*
733 * Enqueue several dmamaps as a single request.
734 */
735/*
736 * Typical usage:
737 *  <queue size> number of followings are stored in arrays
738 *  - command blocks (in dmamem) should be pre-allocated and mapped
739 *  - dmamaps for command blocks should be pre-allocated and loaded
740 *  - dmamaps for payload should be pre-allocated
741 *      r = virtio_enqueue_prep(sc, vq, &slot);		// allocate a slot
742 *	if (r)		// currently 0 or EAGAIN
743 *	  return r;
744 *	r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
745 *	if (r) {
746 *	  virtio_enqueue_abort(sc, vq, slot);
747 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
748 *	  return r;
749 *	}
750 *	r = virtio_enqueue_reserve(sc, vq, slot,
751 *				   dmamap_payload[slot]->dm_nsegs+1);
752 *							// ^ +1 for command
753 *	if (r) {	// currently 0 or EAGAIN
754 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
755 *	  return r;					// do not call abort()
756 *	}
757 *	<setup and prepare commands>
758 *	bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
759 *	bus_dmamap_sync(dmat, dmamap_payload[slot],...);
760 *	virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
761 *	virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
762 *	virtio_enqueue_commit(sc, vq, slot, true);
763 */
764
765/*
766 * enqueue_prep: allocate a slot number
767 */
768int
769virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
770{
771	struct vq_entry *qe1;
772
773	KASSERT(slotp != NULL);
774
775	qe1 = vq_alloc_entry(vq);
776	if (qe1 == NULL)
777		return EAGAIN;
778	/* next slot is not allocated yet */
779	qe1->qe_next = -1;
780	*slotp = qe1->qe_index;
781
782	return 0;
783}
784
785/*
786 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
787 */
788int
789virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
790		       int slot, int nsegs)
791{
792	int indirect;
793	struct vq_entry *qe1 = &vq->vq_entries[slot];
794
795	KASSERT(qe1->qe_next == -1);
796	KASSERT(1 <= nsegs && nsegs <= vq->vq_num);
797
798	if ((vq->vq_indirect != NULL) &&
799	    (nsegs >= MINSEG_INDIRECT) &&
800	    (nsegs <= vq->vq_maxnsegs))
801		indirect = 1;
802	else
803		indirect = 0;
804	qe1->qe_indirect = indirect;
805
806	if (indirect) {
807		struct vring_desc *vd;
808		int i;
809
810		vd = &vq->vq_desc[qe1->qe_index];
811		vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr
812			+ vq->vq_indirectoffset;
813		vd->addr += sizeof(struct vring_desc)
814			* vq->vq_maxnsegs * qe1->qe_index;
815		vd->len = sizeof(struct vring_desc) * nsegs;
816		vd->flags = VRING_DESC_F_INDIRECT;
817
818		vd = vq->vq_indirect;
819		vd += vq->vq_maxnsegs * qe1->qe_index;
820		qe1->qe_desc_base = vd;
821
822		for (i = 0; i < nsegs-1; i++) {
823			vd[i].flags = VRING_DESC_F_NEXT;
824		}
825		vd[i].flags = 0;
826		qe1->qe_next = 0;
827
828		return 0;
829	} else {
830		struct vring_desc *vd;
831		struct vq_entry *qe;
832		int i, s;
833
834		vd = &vq->vq_desc[0];
835		qe1->qe_desc_base = vd;
836		qe1->qe_next = qe1->qe_index;
837		s = slot;
838		for (i = 0; i < nsegs - 1; i++) {
839			qe = vq_alloc_entry(vq);
840			if (qe == NULL) {
841				vd[s].flags = 0;
842				virtio_enqueue_abort(sc, vq, slot);
843				return EAGAIN;
844			}
845			vd[s].flags = VRING_DESC_F_NEXT;
846			vd[s].next = qe->qe_index;
847			s = qe->qe_index;
848		}
849		vd[s].flags = 0;
850
851		return 0;
852	}
853}
854
855/*
856 * enqueue: enqueue a single dmamap.
857 */
858int
859virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
860	       bus_dmamap_t dmamap, bool write)
861{
862	struct vq_entry *qe1 = &vq->vq_entries[slot];
863	struct vring_desc *vd = qe1->qe_desc_base;
864	int i;
865	int s = qe1->qe_next;
866
867	KASSERT(s >= 0);
868	KASSERT(dmamap->dm_nsegs > 0);
869
870	for (i = 0; i < dmamap->dm_nsegs; i++) {
871		vd[s].addr = dmamap->dm_segs[i].ds_addr;
872		vd[s].len = dmamap->dm_segs[i].ds_len;
873		if (!write)
874			vd[s].flags |= VRING_DESC_F_WRITE;
875		s = vd[s].next;
876	}
877	qe1->qe_next = s;
878
879	return 0;
880}
881
882int
883virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
884		 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
885		 bool write)
886{
887	struct vq_entry *qe1 = &vq->vq_entries[slot];
888	struct vring_desc *vd = qe1->qe_desc_base;
889	int s = qe1->qe_next;
890
891	KASSERT(s >= 0);
892	KASSERT(dmamap->dm_nsegs == 1); /* XXX */
893	KASSERT((dmamap->dm_segs[0].ds_len > start) &&
894		(dmamap->dm_segs[0].ds_len >= start + len));
895
896	vd[s].addr = dmamap->dm_segs[0].ds_addr + start;
897	vd[s].len = len;
898	if (!write)
899		vd[s].flags |= VRING_DESC_F_WRITE;
900	qe1->qe_next = vd[s].next;
901
902	return 0;
903}
904
905/*
906 * enqueue_commit: add it to the aring.
907 */
908int
909virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
910		      bool notifynow)
911{
912	struct vq_entry *qe1;
913
914	if (slot < 0) {
915		mutex_enter(&vq->vq_aring_lock);
916		goto notify;
917	}
918	vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
919	qe1 = &vq->vq_entries[slot];
920	if (qe1->qe_indirect)
921		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
922	mutex_enter(&vq->vq_aring_lock);
923	vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot;
924
925notify:
926	if (notifynow) {
927		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
928		vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
929		membar_producer();
930		vq->vq_avail->idx = vq->vq_avail_idx;
931		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
932		membar_producer();
933		vq->vq_queued++;
934		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
935		membar_consumer();
936		if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY))
937			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
938					  VIRTIO_CONFIG_QUEUE_NOTIFY,
939					  vq->vq_index);
940	}
941	mutex_exit(&vq->vq_aring_lock);
942
943	return 0;
944}
945
946/*
947 * enqueue_abort: rollback.
948 */
949int
950virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
951{
952	struct vq_entry *qe = &vq->vq_entries[slot];
953	struct vring_desc *vd;
954	int s;
955
956	if (qe->qe_next < 0) {
957		vq_free_entry(vq, qe);
958		return 0;
959	}
960
961	s = slot;
962	vd = &vq->vq_desc[0];
963	while (vd[s].flags & VRING_DESC_F_NEXT) {
964		s = vd[s].next;
965		vq_free_entry(vq, qe);
966		qe = &vq->vq_entries[s];
967	}
968	vq_free_entry(vq, qe);
969	return 0;
970}
971
972/*
973 * Dequeue a request.
974 */
975/*
976 * dequeue: dequeue a request from uring; dmamap_sync for uring is
977 *	    already done in the interrupt handler.
978 */
979int
980virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
981	       int *slotp, int *lenp)
982{
983	uint16_t slot, usedidx;
984	struct vq_entry *qe;
985
986	if (vq->vq_used_idx == vq->vq_used->idx)
987		return ENOENT;
988	mutex_enter(&vq->vq_uring_lock);
989	usedidx = vq->vq_used_idx++;
990	mutex_exit(&vq->vq_uring_lock);
991	usedidx %= vq->vq_num;
992	slot = vq->vq_used->ring[usedidx].id;
993	qe = &vq->vq_entries[slot];
994
995	if (qe->qe_indirect)
996		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
997
998	if (slotp)
999		*slotp = slot;
1000	if (lenp)
1001		*lenp = vq->vq_used->ring[usedidx].len;
1002
1003	return 0;
1004}
1005
1006/*
1007 * dequeue_commit: complete dequeue; the slot is recycled for future use.
1008 *                 if you forget to call this the slot will be leaked.
1009 */
1010int
1011virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
1012{
1013	struct vq_entry *qe = &vq->vq_entries[slot];
1014	struct vring_desc *vd = &vq->vq_desc[0];
1015	int s = slot;
1016
1017	while (vd[s].flags & VRING_DESC_F_NEXT) {
1018		s = vd[s].next;
1019		vq_free_entry(vq, qe);
1020		qe = &vq->vq_entries[s];
1021	}
1022	vq_free_entry(vq, qe);
1023
1024	return 0;
1025}
1026