virtio.c revision 1.31 
1/*	$NetBSD: virtio.c,v 1.31 2018/06/02 22:43:15 jakllsch Exp $	*/
2
3/*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.31 2018/06/02 22:43:15 jakllsch Exp $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/atomic.h>
35#include <sys/bus.h>
36#include <sys/device.h>
37#include <sys/kmem.h>
38#include <sys/module.h>
39
40#include <dev/pci/pcidevs.h>
41#include <dev/pci/pcireg.h>
42#include <dev/pci/pcivar.h>
43
44#define VIRTIO_PRIVATE
45
46#include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
47#include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
48
49#define MINSEG_INDIRECT		2 /* use indirect if nsegs >= this value */
50
51static void	virtio_init_vq(struct virtio_softc *,
52		    struct virtqueue *, const bool);
53
54void
55virtio_set_status(struct virtio_softc *sc, int status)
56{
57	sc->sc_ops->set_status(sc, status);
58}
59
60/*
61 * Reset the device.
62 */
63/*
64 * To reset the device to a known state, do following:
65 *	virtio_reset(sc);	     // this will stop the device activity
66 *	<dequeue finished requests>; // virtio_dequeue() still can be called
67 *	<revoke pending requests in the vqs if any>;
68 *	virtio_reinit_begin(sc);     // dequeue prohibitted
69 *	newfeatures = virtio_negotiate_features(sc, requestedfeatures);
70 *	<some other initialization>;
71 *	virtio_reinit_end(sc);	     // device activated; enqueue allowed
72 * Once attached, feature negotiation can only be allowed after virtio_reset.
73 */
74void
75virtio_reset(struct virtio_softc *sc)
76{
77	virtio_device_reset(sc);
78}
79
80void
81virtio_reinit_start(struct virtio_softc *sc)
82{
83	int i;
84
85	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
86	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
87	for (i = 0; i < sc->sc_nvqs; i++) {
88		int n;
89		struct virtqueue *vq = &sc->sc_vqs[i];
90		n = sc->sc_ops->read_queue_size(sc, vq->vq_index);
91		if (n == 0)	/* vq disappeared */
92			continue;
93		if (n != vq->vq_num) {
94			panic("%s: virtqueue size changed, vq index %d\n",
95			      device_xname(sc->sc_dev),
96			      vq->vq_index);
97		}
98		virtio_init_vq(sc, vq, true);
99		sc->sc_ops->setup_queue(sc, vq->vq_index,
100		    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
101	}
102}
103
104void
105virtio_reinit_end(struct virtio_softc *sc)
106{
107	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
108}
109
110/*
111 * Feature negotiation.
112 */
113uint32_t
114virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
115{
116	uint32_t r;
117
118	if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
119	    !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
120		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
121	r = sc->sc_ops->neg_features(sc, guest_features);
122	sc->sc_features = r;
123	if (r & VIRTIO_F_RING_INDIRECT_DESC)
124		sc->sc_indirect = true;
125	else
126		sc->sc_indirect = false;
127
128	return r;
129}
130
131/*
132 * Device configuration registers.
133 */
134uint8_t
135virtio_read_device_config_1(struct virtio_softc *sc, int index)
136{
137	return sc->sc_ops->read_dev_cfg_1(sc, index);
138}
139
140uint16_t
141virtio_read_device_config_2(struct virtio_softc *sc, int index)
142{
143	return sc->sc_ops->read_dev_cfg_2(sc, index);
144}
145
146uint32_t
147virtio_read_device_config_4(struct virtio_softc *sc, int index)
148{
149	return sc->sc_ops->read_dev_cfg_4(sc, index);
150}
151
152uint64_t
153virtio_read_device_config_8(struct virtio_softc *sc, int index)
154{
155	return sc->sc_ops->read_dev_cfg_8(sc, index);
156}
157
158void
159virtio_write_device_config_1(struct virtio_softc *sc,
160			     int index, uint8_t value)
161{
162	return sc->sc_ops->write_dev_cfg_1(sc, index, value);
163}
164
165void
166virtio_write_device_config_2(struct virtio_softc *sc,
167			     int index, uint16_t value)
168{
169	return sc->sc_ops->write_dev_cfg_2(sc, index, value);
170}
171
172void
173virtio_write_device_config_4(struct virtio_softc *sc,
174			     int index, uint32_t value)
175{
176	return sc->sc_ops->write_dev_cfg_4(sc, index, value);
177}
178
179void
180virtio_write_device_config_8(struct virtio_softc *sc,
181			     int index, uint64_t value)
182{
183	return sc->sc_ops->write_dev_cfg_8(sc, index, value);
184}
185
186/*
187 * Interrupt handler.
188 */
189static void
190virtio_soft_intr(void *arg)
191{
192	struct virtio_softc *sc = arg;
193
194	KASSERT(sc->sc_intrhand != NULL);
195
196	(sc->sc_intrhand)(sc);
197}
198
199/*
200 * dmamap sync operations for a virtqueue.
201 */
202static inline void
203vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
204{
205	/* availoffset == sizeof(vring_desc)*vq_num */
206	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
207			ops);
208}
209
210static inline void
211vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
212{
213	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
214			vq->vq_availoffset,
215			offsetof(struct vring_avail, ring)
216			 + vq->vq_num * sizeof(uint16_t),
217			ops);
218}
219
220static inline void
221vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
222{
223	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
224			vq->vq_usedoffset,
225			offsetof(struct vring_used, ring)
226			 + vq->vq_num * sizeof(struct vring_used_elem),
227			ops);
228}
229
230static inline void
231vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
232		     int ops)
233{
234	int offset = vq->vq_indirectoffset
235		      + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
236
237	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
238			offset, sizeof(struct vring_desc) * vq->vq_maxnsegs,
239			ops);
240}
241
242/*
243 * Can be used as sc_intrhand.
244 */
245/*
246 * Scan vq, bus_dmamap_sync for the vqs (not for the payload),
247 * and calls (*vq_done)() if some entries are consumed.
248 */
249int
250virtio_vq_intr(struct virtio_softc *sc)
251{
252	struct virtqueue *vq;
253	int i, r = 0;
254
255	for (i = 0; i < sc->sc_nvqs; i++) {
256		vq = &sc->sc_vqs[i];
257		if (vq->vq_queued) {
258			vq->vq_queued = 0;
259			vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
260		}
261		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
262		membar_consumer();
263		if (vq->vq_used_idx != vq->vq_used->idx) {
264			if (vq->vq_done)
265				r |= (vq->vq_done)(vq);
266		}
267	}
268
269	return r;
270}
271
272/*
273 * Start/stop vq interrupt.  No guarantee.
274 */
275void
276virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
277{
278	vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
279	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
280	vq->vq_queued++;
281}
282
283void
284virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
285{
286	vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
287	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
288	vq->vq_queued++;
289}
290
291/*
292 * Initialize vq structure.
293 */
294static void
295virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq,
296    const bool reinit)
297{
298	int i, j;
299	int vq_size = vq->vq_num;
300
301	memset(vq->vq_vaddr, 0, vq->vq_bytesize);
302
303	/* build the indirect descriptor chain */
304	if (vq->vq_indirect != NULL) {
305		struct vring_desc *vd;
306
307		for (i = 0; i < vq_size; i++) {
308			vd = vq->vq_indirect;
309			vd += vq->vq_maxnsegs * i;
310			for (j = 0; j < vq->vq_maxnsegs-1; j++) {
311				vd[j].next = j + 1;
312			}
313		}
314	}
315
316	/* free slot management */
317	SIMPLEQ_INIT(&vq->vq_freelist);
318	for (i = 0; i < vq_size; i++) {
319		SIMPLEQ_INSERT_TAIL(&vq->vq_freelist,
320				    &vq->vq_entries[i], qe_list);
321		vq->vq_entries[i].qe_index = i;
322	}
323	if (!reinit)
324		mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl);
325
326	/* enqueue/dequeue status */
327	vq->vq_avail_idx = 0;
328	vq->vq_used_idx = 0;
329	vq->vq_queued = 0;
330	if (!reinit) {
331		mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
332		mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
333	}
334	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
335	vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
336	vq->vq_queued++;
337}
338
339/*
340 * Allocate/free a vq.
341 */
342int
343virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index,
344    int maxsegsize, int maxnsegs, const char *name)
345{
346	int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0;
347	int rsegs, r;
348#define VIRTQUEUE_ALIGN(n)	(((n)+(VIRTIO_PAGE_SIZE-1))&	\
349				 ~(VIRTIO_PAGE_SIZE-1))
350
351	/* Make sure callers allocate vqs in order */
352	KASSERT(sc->sc_nvqs == index);
353
354	memset(vq, 0, sizeof(*vq));
355
356	vq_size = sc->sc_ops->read_queue_size(sc, index);
357	if (vq_size == 0) {
358		aprint_error_dev(sc->sc_dev,
359				 "virtqueue not exist, index %d for %s\n",
360				 index, name);
361		goto err;
362	}
363	/* allocsize1: descriptor table + avail ring + pad */
364	allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size
365				     + sizeof(uint16_t)*(2+vq_size));
366	/* allocsize2: used ring + pad */
367	allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2
368				     + sizeof(struct vring_used_elem)*vq_size);
369	/* allocsize3: indirect table */
370	if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT)
371		allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size;
372	else
373		allocsize3 = 0;
374	allocsize = allocsize1 + allocsize2 + allocsize3;
375
376	/* alloc and map the memory */
377	r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
378			     &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
379	if (r != 0) {
380		aprint_error_dev(sc->sc_dev,
381				 "virtqueue %d for %s allocation failed, "
382				 "error code %d\n", index, name, r);
383		goto err;
384	}
385	r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize,
386			   &vq->vq_vaddr, BUS_DMA_NOWAIT);
387	if (r != 0) {
388		aprint_error_dev(sc->sc_dev,
389				 "virtqueue %d for %s map failed, "
390				 "error code %d\n", index, name, r);
391		goto err;
392	}
393	r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
394			      BUS_DMA_NOWAIT, &vq->vq_dmamap);
395	if (r != 0) {
396		aprint_error_dev(sc->sc_dev,
397				 "virtqueue %d for %s dmamap creation failed, "
398				 "error code %d\n", index, name, r);
399		goto err;
400	}
401	r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
402			    vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT);
403	if (r != 0) {
404		aprint_error_dev(sc->sc_dev,
405				 "virtqueue %d for %s dmamap load failed, "
406				 "error code %d\n", index, name, r);
407		goto err;
408	}
409
410	/* set the vq address */
411	sc->sc_ops->setup_queue(sc, vq->vq_index,
412	    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
413
414	/* remember addresses and offsets for later use */
415	vq->vq_owner = sc;
416	vq->vq_num = vq_size;
417	vq->vq_index = index;
418	vq->vq_desc = vq->vq_vaddr;
419	vq->vq_availoffset = sizeof(struct vring_desc)*vq_size;
420	vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset);
421	vq->vq_usedoffset = allocsize1;
422	vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset);
423	if (allocsize3 > 0) {
424		vq->vq_indirectoffset = allocsize1 + allocsize2;
425		vq->vq_indirect = (void*)(((char*)vq->vq_desc)
426					  + vq->vq_indirectoffset);
427	}
428	vq->vq_bytesize = allocsize;
429	vq->vq_maxsegsize = maxsegsize;
430	vq->vq_maxnsegs = maxnsegs;
431
432	/* free slot management */
433	vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size,
434				     KM_NOSLEEP);
435	if (vq->vq_entries == NULL) {
436		r = ENOMEM;
437		goto err;
438	}
439
440	virtio_init_vq(sc, vq, false);
441
442	aprint_verbose_dev(sc->sc_dev,
443			   "allocated %u byte for virtqueue %d for %s, "
444			   "size %d\n", allocsize, index, name, vq_size);
445	if (allocsize3 > 0)
446		aprint_verbose_dev(sc->sc_dev,
447				   "using %d byte (%d entries) "
448				   "indirect descriptors\n",
449				   allocsize3, maxnsegs * vq_size);
450
451	sc->sc_nvqs++;
452
453	return 0;
454
455err:
456	sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
457	if (vq->vq_dmamap)
458		bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
459	if (vq->vq_vaddr)
460		bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
461	if (vq->vq_segs[0].ds_addr)
462		bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
463	memset(vq, 0, sizeof(*vq));
464
465	return -1;
466}
467
468int
469virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
470{
471	struct vq_entry *qe;
472	int i = 0;
473
474	/* device must be already deactivated */
475	/* confirm the vq is empty */
476	SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) {
477		i++;
478	}
479	if (i != vq->vq_num) {
480		printf("%s: freeing non-empty vq, index %d\n",
481		       device_xname(sc->sc_dev), vq->vq_index);
482		return EBUSY;
483	}
484
485	/* tell device that there's no virtqueue any longer */
486	sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
487
488	kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num);
489	bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
490	bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
491	bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
492	bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
493	mutex_destroy(&vq->vq_freelist_lock);
494	mutex_destroy(&vq->vq_uring_lock);
495	mutex_destroy(&vq->vq_aring_lock);
496	memset(vq, 0, sizeof(*vq));
497
498	sc->sc_nvqs--;
499
500	return 0;
501}
502
503/*
504 * Free descriptor management.
505 */
506static struct vq_entry *
507vq_alloc_entry(struct virtqueue *vq)
508{
509	struct vq_entry *qe;
510
511	mutex_enter(&vq->vq_freelist_lock);
512	if (SIMPLEQ_EMPTY(&vq->vq_freelist)) {
513		mutex_exit(&vq->vq_freelist_lock);
514		return NULL;
515	}
516	qe = SIMPLEQ_FIRST(&vq->vq_freelist);
517	SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list);
518	mutex_exit(&vq->vq_freelist_lock);
519
520	return qe;
521}
522
523static void
524vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
525{
526	mutex_enter(&vq->vq_freelist_lock);
527	SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list);
528	mutex_exit(&vq->vq_freelist_lock);
529
530	return;
531}
532
533/*
534 * Enqueue several dmamaps as a single request.
535 */
536/*
537 * Typical usage:
538 *  <queue size> number of followings are stored in arrays
539 *  - command blocks (in dmamem) should be pre-allocated and mapped
540 *  - dmamaps for command blocks should be pre-allocated and loaded
541 *  - dmamaps for payload should be pre-allocated
542 *      r = virtio_enqueue_prep(sc, vq, &slot);		// allocate a slot
543 *	if (r)		// currently 0 or EAGAIN
544 *	  return r;
545 *	r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
546 *	if (r) {
547 *	  virtio_enqueue_abort(sc, vq, slot);
548 *	  return r;
549 *	}
550 *	r = virtio_enqueue_reserve(sc, vq, slot,
551 *				   dmamap_payload[slot]->dm_nsegs+1);
552 *							// ^ +1 for command
553 *	if (r) {	// currently 0 or EAGAIN
554 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
555 *	  return r;					// do not call abort()
556 *	}
557 *	<setup and prepare commands>
558 *	bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
559 *	bus_dmamap_sync(dmat, dmamap_payload[slot],...);
560 *	virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
561 *	virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
562 *	virtio_enqueue_commit(sc, vq, slot, true);
563 */
564
565/*
566 * enqueue_prep: allocate a slot number
567 */
568int
569virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
570{
571	struct vq_entry *qe1;
572
573	KASSERT(slotp != NULL);
574
575	qe1 = vq_alloc_entry(vq);
576	if (qe1 == NULL)
577		return EAGAIN;
578	/* next slot is not allocated yet */
579	qe1->qe_next = -1;
580	*slotp = qe1->qe_index;
581
582	return 0;
583}
584
585/*
586 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
587 */
588int
589virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
590		       int slot, int nsegs)
591{
592	int indirect;
593	struct vq_entry *qe1 = &vq->vq_entries[slot];
594
595	KASSERT(qe1->qe_next == -1);
596	KASSERT(1 <= nsegs && nsegs <= vq->vq_num);
597
598	if ((vq->vq_indirect != NULL) &&
599	    (nsegs >= MINSEG_INDIRECT) &&
600	    (nsegs <= vq->vq_maxnsegs))
601		indirect = 1;
602	else
603		indirect = 0;
604	qe1->qe_indirect = indirect;
605
606	if (indirect) {
607		struct vring_desc *vd;
608		int i;
609
610		vd = &vq->vq_desc[qe1->qe_index];
611		vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr
612			+ vq->vq_indirectoffset;
613		vd->addr += sizeof(struct vring_desc)
614			* vq->vq_maxnsegs * qe1->qe_index;
615		vd->len = sizeof(struct vring_desc) * nsegs;
616		vd->flags = VRING_DESC_F_INDIRECT;
617
618		vd = vq->vq_indirect;
619		vd += vq->vq_maxnsegs * qe1->qe_index;
620		qe1->qe_desc_base = vd;
621
622		for (i = 0; i < nsegs-1; i++) {
623			vd[i].flags = VRING_DESC_F_NEXT;
624		}
625		vd[i].flags = 0;
626		qe1->qe_next = 0;
627
628		return 0;
629	} else {
630		struct vring_desc *vd;
631		struct vq_entry *qe;
632		int i, s;
633
634		vd = &vq->vq_desc[0];
635		qe1->qe_desc_base = vd;
636		qe1->qe_next = qe1->qe_index;
637		s = slot;
638		for (i = 0; i < nsegs - 1; i++) {
639			qe = vq_alloc_entry(vq);
640			if (qe == NULL) {
641				vd[s].flags = 0;
642				virtio_enqueue_abort(sc, vq, slot);
643				return EAGAIN;
644			}
645			vd[s].flags = VRING_DESC_F_NEXT;
646			vd[s].next = qe->qe_index;
647			s = qe->qe_index;
648		}
649		vd[s].flags = 0;
650
651		return 0;
652	}
653}
654
655/*
656 * enqueue: enqueue a single dmamap.
657 */
658int
659virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
660	       bus_dmamap_t dmamap, bool write)
661{
662	struct vq_entry *qe1 = &vq->vq_entries[slot];
663	struct vring_desc *vd = qe1->qe_desc_base;
664	int i;
665	int s = qe1->qe_next;
666
667	KASSERT(s >= 0);
668	KASSERT(dmamap->dm_nsegs > 0);
669
670	for (i = 0; i < dmamap->dm_nsegs; i++) {
671		vd[s].addr = dmamap->dm_segs[i].ds_addr;
672		vd[s].len = dmamap->dm_segs[i].ds_len;
673		if (!write)
674			vd[s].flags |= VRING_DESC_F_WRITE;
675		s = vd[s].next;
676	}
677	qe1->qe_next = s;
678
679	return 0;
680}
681
682int
683virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
684		 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
685		 bool write)
686{
687	struct vq_entry *qe1 = &vq->vq_entries[slot];
688	struct vring_desc *vd = qe1->qe_desc_base;
689	int s = qe1->qe_next;
690
691	KASSERT(s >= 0);
692	KASSERT(dmamap->dm_nsegs == 1); /* XXX */
693	KASSERT((dmamap->dm_segs[0].ds_len > start) &&
694		(dmamap->dm_segs[0].ds_len >= start + len));
695
696	vd[s].addr = dmamap->dm_segs[0].ds_addr + start;
697	vd[s].len = len;
698	if (!write)
699		vd[s].flags |= VRING_DESC_F_WRITE;
700	qe1->qe_next = vd[s].next;
701
702	return 0;
703}
704
705/*
706 * enqueue_commit: add it to the aring.
707 */
708int
709virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
710		      bool notifynow)
711{
712	struct vq_entry *qe1;
713
714	if (slot < 0) {
715		mutex_enter(&vq->vq_aring_lock);
716		goto notify;
717	}
718	vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
719	qe1 = &vq->vq_entries[slot];
720	if (qe1->qe_indirect)
721		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
722	mutex_enter(&vq->vq_aring_lock);
723	vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot;
724
725notify:
726	if (notifynow) {
727		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
728		vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
729		membar_producer();
730		vq->vq_avail->idx = vq->vq_avail_idx;
731		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
732		membar_producer();
733		vq->vq_queued++;
734		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
735		membar_consumer();
736		if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY))
737			sc->sc_ops->kick(sc, vq->vq_index);
738	}
739	mutex_exit(&vq->vq_aring_lock);
740
741	return 0;
742}
743
744/*
745 * enqueue_abort: rollback.
746 */
747int
748virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
749{
750	struct vq_entry *qe = &vq->vq_entries[slot];
751	struct vring_desc *vd;
752	int s;
753
754	if (qe->qe_next < 0) {
755		vq_free_entry(vq, qe);
756		return 0;
757	}
758
759	s = slot;
760	vd = &vq->vq_desc[0];
761	while (vd[s].flags & VRING_DESC_F_NEXT) {
762		s = vd[s].next;
763		vq_free_entry(vq, qe);
764		qe = &vq->vq_entries[s];
765	}
766	vq_free_entry(vq, qe);
767	return 0;
768}
769
770/*
771 * Dequeue a request.
772 */
773/*
774 * dequeue: dequeue a request from uring; dmamap_sync for uring is
775 *	    already done in the interrupt handler.
776 */
777int
778virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
779	       int *slotp, int *lenp)
780{
781	uint16_t slot, usedidx;
782	struct vq_entry *qe;
783
784	if (vq->vq_used_idx == vq->vq_used->idx)
785		return ENOENT;
786	mutex_enter(&vq->vq_uring_lock);
787	usedidx = vq->vq_used_idx++;
788	mutex_exit(&vq->vq_uring_lock);
789	usedidx %= vq->vq_num;
790	slot = vq->vq_used->ring[usedidx].id;
791	qe = &vq->vq_entries[slot];
792
793	if (qe->qe_indirect)
794		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
795
796	if (slotp)
797		*slotp = slot;
798	if (lenp)
799		*lenp = vq->vq_used->ring[usedidx].len;
800
801	return 0;
802}
803
804/*
805 * dequeue_commit: complete dequeue; the slot is recycled for future use.
806 *                 if you forget to call this the slot will be leaked.
807 */
808int
809virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
810{
811	struct vq_entry *qe = &vq->vq_entries[slot];
812	struct vring_desc *vd = &vq->vq_desc[0];
813	int s = slot;
814
815	while (vd[s].flags & VRING_DESC_F_NEXT) {
816		s = vd[s].next;
817		vq_free_entry(vq, qe);
818		qe = &vq->vq_entries[s];
819	}
820	vq_free_entry(vq, qe);
821
822	return 0;
823}
824
825/*
826 * Attach a child, fill all the members.
827 */
828void
829virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl,
830		    struct virtqueue *vqs,
831		    virtio_callback config_change,
832		    virtio_callback intr_hand,
833		    int req_flags, int req_features, const char *feat_bits)
834{
835	char buf[256];
836	int features;
837
838	sc->sc_child = child;
839	sc->sc_ipl = ipl;
840	sc->sc_vqs = vqs;
841	sc->sc_config_change = config_change;
842	sc->sc_intrhand = intr_hand;
843	sc->sc_flags = req_flags;
844
845	features = virtio_negotiate_features(sc, req_features);
846	snprintb(buf, sizeof(buf), feat_bits, features);
847	aprint_normal(": Features: %s\n", buf);
848	aprint_naive("\n");
849}
850
851int
852virtio_child_attach_finish(struct virtio_softc *sc)
853{
854	int r;
855
856	r = sc->sc_ops->setup_interrupts(sc);
857	if (r != 0) {
858		aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n");
859		goto fail;
860	}
861
862	KASSERT(sc->sc_soft_ih == NULL);
863	if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) {
864		u_int flags = SOFTINT_NET;
865		if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
866			flags |= SOFTINT_MPSAFE;
867
868		sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc);
869		if (sc->sc_soft_ih == NULL) {
870			sc->sc_ops->free_interrupts(sc);
871			aprint_error_dev(sc->sc_dev,
872			    "failed to establish soft interrupt\n");
873			goto fail;
874		}
875	}
876
877	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
878	return 0;
879
880fail:
881	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
882	return 1;
883}
884
885void
886virtio_child_detach(struct virtio_softc *sc)
887{
888	sc->sc_child = NULL;
889	sc->sc_vqs = NULL;
890
891	virtio_device_reset(sc);
892
893	sc->sc_ops->free_interrupts(sc);
894
895	if (sc->sc_soft_ih) {
896		softint_disestablish(sc->sc_soft_ih);
897		sc->sc_soft_ih = NULL;
898	}
899}
900
901void
902virtio_child_attach_failed(struct virtio_softc *sc)
903{
904	virtio_child_detach(sc);
905
906	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
907
908	sc->sc_child = VIRTIO_CHILD_FAILED;
909}
910
911bus_dma_tag_t
912virtio_dmat(struct virtio_softc *sc)
913{
914	return sc->sc_dmat;
915}
916
917device_t
918virtio_child(struct virtio_softc *sc)
919{
920	return sc->sc_child;
921}
922
923int
924virtio_intrhand(struct virtio_softc *sc)
925{
926	return (sc->sc_intrhand)(sc);
927}
928
929uint32_t
930virtio_features(struct virtio_softc *sc)
931{
932	return sc->sc_features;
933}
934
935MODULE(MODULE_CLASS_DRIVER, virtio, "pci");
936
937#ifdef _MODULE
938#include "ioconf.c"
939#endif
940
941static int
942virtio_modcmd(modcmd_t cmd, void *opaque)
943{
944	int error = 0;
945
946#ifdef _MODULE
947	switch (cmd) {
948	case MODULE_CMD_INIT:
949		error = config_init_component(cfdriver_ioconf_virtio,
950		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
951		break;
952	case MODULE_CMD_FINI:
953		error = config_fini_component(cfdriver_ioconf_virtio,
954		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
955		break;
956	default:
957		error = ENOTTY;
958		break;
959	}
960#endif
961
962	return error;
963}
964