virtio.c revision 1.42 
1/*	$NetBSD: virtio.c,v 1.42 2020/09/17 17:09:59 jakllsch Exp $	*/
2
3/*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.42 2020/09/17 17:09:59 jakllsch Exp $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/atomic.h>
35#include <sys/bus.h>
36#include <sys/device.h>
37#include <sys/kmem.h>
38#include <sys/module.h>
39
40#define VIRTIO_PRIVATE
41
42#include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
43#include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
44
45#define MINSEG_INDIRECT		2 /* use indirect if nsegs >= this value */
46
47static void	virtio_init_vq(struct virtio_softc *,
48		    struct virtqueue *, const bool);
49
50void
51virtio_set_status(struct virtio_softc *sc, int status)
52{
53	sc->sc_ops->set_status(sc, status);
54}
55
56/*
57 * Reset the device.
58 */
59/*
60 * To reset the device to a known state, do following:
61 *	virtio_reset(sc);	     // this will stop the device activity
62 *	<dequeue finished requests>; // virtio_dequeue() still can be called
63 *	<revoke pending requests in the vqs if any>;
64 *	virtio_reinit_start(sc);     // dequeue prohibitted
65 *	newfeatures = virtio_negotiate_features(sc, requestedfeatures);
66 *	<some other initialization>;
67 *	virtio_reinit_end(sc);	     // device activated; enqueue allowed
68 * Once attached, feature negotiation can only be allowed after virtio_reset.
69 */
70void
71virtio_reset(struct virtio_softc *sc)
72{
73	virtio_device_reset(sc);
74}
75
76void
77virtio_reinit_start(struct virtio_softc *sc)
78{
79	int i;
80
81	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
82	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
83	for (i = 0; i < sc->sc_nvqs; i++) {
84		int n;
85		struct virtqueue *vq = &sc->sc_vqs[i];
86		n = sc->sc_ops->read_queue_size(sc, vq->vq_index);
87		if (n == 0)	/* vq disappeared */
88			continue;
89		if (n != vq->vq_num) {
90			panic("%s: virtqueue size changed, vq index %d\n",
91			      device_xname(sc->sc_dev),
92			      vq->vq_index);
93		}
94		virtio_init_vq(sc, vq, true);
95		sc->sc_ops->setup_queue(sc, vq->vq_index,
96		    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
97	}
98}
99
100void
101virtio_reinit_end(struct virtio_softc *sc)
102{
103	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
104}
105
106/*
107 * Feature negotiation.
108 */
109uint32_t
110virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
111{
112	uint32_t r;
113
114	if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
115	    !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
116		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
117	r = sc->sc_ops->neg_features(sc, guest_features);
118	sc->sc_features = r;
119	if (r & VIRTIO_F_RING_INDIRECT_DESC)
120		sc->sc_indirect = true;
121	else
122		sc->sc_indirect = false;
123
124	return r;
125}
126
127/*
128 * Device configuration registers.
129 */
130uint8_t
131virtio_read_device_config_1(struct virtio_softc *sc, int index)
132{
133	return sc->sc_ops->read_dev_cfg_1(sc, index);
134}
135
136uint16_t
137virtio_read_device_config_2(struct virtio_softc *sc, int index)
138{
139	return sc->sc_ops->read_dev_cfg_2(sc, index);
140}
141
142uint32_t
143virtio_read_device_config_4(struct virtio_softc *sc, int index)
144{
145	return sc->sc_ops->read_dev_cfg_4(sc, index);
146}
147
148uint64_t
149virtio_read_device_config_8(struct virtio_softc *sc, int index)
150{
151	return sc->sc_ops->read_dev_cfg_8(sc, index);
152}
153
154void
155virtio_write_device_config_1(struct virtio_softc *sc,
156			     int index, uint8_t value)
157{
158	return sc->sc_ops->write_dev_cfg_1(sc, index, value);
159}
160
161void
162virtio_write_device_config_2(struct virtio_softc *sc,
163			     int index, uint16_t value)
164{
165	return sc->sc_ops->write_dev_cfg_2(sc, index, value);
166}
167
168void
169virtio_write_device_config_4(struct virtio_softc *sc,
170			     int index, uint32_t value)
171{
172	return sc->sc_ops->write_dev_cfg_4(sc, index, value);
173}
174
175void
176virtio_write_device_config_8(struct virtio_softc *sc,
177			     int index, uint64_t value)
178{
179	return sc->sc_ops->write_dev_cfg_8(sc, index, value);
180}
181
182/*
183 * Interrupt handler.
184 */
185static void
186virtio_soft_intr(void *arg)
187{
188	struct virtio_softc *sc = arg;
189
190	KASSERT(sc->sc_intrhand != NULL);
191
192	(sc->sc_intrhand)(sc);
193}
194
195/*
196 * dmamap sync operations for a virtqueue.
197 */
198static inline void
199vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
200{
201	/* availoffset == sizeof(vring_desc)*vq_num */
202	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
203			ops);
204}
205
206static inline void
207vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
208{
209	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
210			vq->vq_availoffset,
211			offsetof(struct vring_avail, ring)
212			 + vq->vq_num * sizeof(uint16_t),
213			ops);
214}
215
216static inline void
217vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
218{
219	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
220			vq->vq_usedoffset,
221			offsetof(struct vring_used, ring)
222			 + vq->vq_num * sizeof(struct vring_used_elem),
223			ops);
224}
225
226static inline void
227vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
228		     int ops)
229{
230	int offset = vq->vq_indirectoffset
231		      + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
232
233	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
234			offset, sizeof(struct vring_desc) * vq->vq_maxnsegs,
235			ops);
236}
237
238/*
239 * Can be used as sc_intrhand.
240 */
241/*
242 * Scan vq, bus_dmamap_sync for the vqs (not for the payload),
243 * and calls (*vq_done)() if some entries are consumed.
244 */
245bool
246virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq)
247{
248
249	if (vq->vq_queued) {
250		vq->vq_queued = 0;
251		vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
252	}
253	vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
254	membar_consumer();
255
256	return (vq->vq_used_idx != vq->vq_used->idx) ? 1 : 0;
257}
258
259int
260virtio_vq_intr(struct virtio_softc *sc)
261{
262	struct virtqueue *vq;
263	int i, r = 0;
264
265	for (i = 0; i < sc->sc_nvqs; i++) {
266		vq = &sc->sc_vqs[i];
267		if (virtio_vq_is_enqueued(sc, vq) == 1) {
268			if (vq->vq_done)
269				r |= (vq->vq_done)(vq);
270		}
271	}
272
273	return r;
274}
275
276int
277virtio_vq_intrhand(struct virtio_softc *sc)
278{
279	struct virtqueue *vq;
280	int i, r = 0;
281
282	for (i = 0; i < sc->sc_nvqs; i++) {
283		vq = &sc->sc_vqs[i];
284		r |= (vq->vq_intrhand)(vq->vq_intrhand_arg);
285	}
286
287	return r;
288}
289
290/*
291 * Start/stop vq interrupt.  No guarantee.
292 */
293void
294virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
295{
296	vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
297	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
298	vq->vq_queued++;
299}
300
301void
302virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
303{
304	vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
305	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
306	vq->vq_queued++;
307}
308
309/*
310 * Initialize vq structure.
311 */
312static void
313virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq,
314    const bool reinit)
315{
316	int i, j;
317	int vq_size = vq->vq_num;
318
319	memset(vq->vq_vaddr, 0, vq->vq_bytesize);
320
321	/* build the indirect descriptor chain */
322	if (vq->vq_indirect != NULL) {
323		struct vring_desc *vd;
324
325		for (i = 0; i < vq_size; i++) {
326			vd = vq->vq_indirect;
327			vd += vq->vq_maxnsegs * i;
328			for (j = 0; j < vq->vq_maxnsegs-1; j++) {
329				vd[j].next = j + 1;
330			}
331		}
332	}
333
334	/* free slot management */
335	SIMPLEQ_INIT(&vq->vq_freelist);
336	for (i = 0; i < vq_size; i++) {
337		SIMPLEQ_INSERT_TAIL(&vq->vq_freelist,
338				    &vq->vq_entries[i], qe_list);
339		vq->vq_entries[i].qe_index = i;
340	}
341	if (!reinit)
342		mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl);
343
344	/* enqueue/dequeue status */
345	vq->vq_avail_idx = 0;
346	vq->vq_used_idx = 0;
347	vq->vq_queued = 0;
348	if (!reinit) {
349		mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
350		mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
351	}
352	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
353	vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
354	vq->vq_queued++;
355}
356
357/*
358 * Allocate/free a vq.
359 */
360int
361virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index,
362    int maxsegsize, int maxnsegs, const char *name)
363{
364	int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0;
365	int rsegs, r;
366#define VIRTQUEUE_ALIGN(n)	(((n)+(VIRTIO_PAGE_SIZE-1))&	\
367				 ~(VIRTIO_PAGE_SIZE-1))
368
369	/* Make sure callers allocate vqs in order */
370	KASSERT(sc->sc_nvqs == index);
371
372	memset(vq, 0, sizeof(*vq));
373
374	vq_size = sc->sc_ops->read_queue_size(sc, index);
375	if (vq_size == 0) {
376		aprint_error_dev(sc->sc_dev,
377				 "virtqueue not exist, index %d for %s\n",
378				 index, name);
379		goto err;
380	}
381	/* allocsize1: descriptor table + avail ring + pad */
382	allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size
383				     + sizeof(uint16_t)*(2+vq_size));
384	/* allocsize2: used ring + pad */
385	allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2
386				     + sizeof(struct vring_used_elem)*vq_size);
387	/* allocsize3: indirect table */
388	if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT)
389		allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size;
390	else
391		allocsize3 = 0;
392	allocsize = allocsize1 + allocsize2 + allocsize3;
393
394	/* alloc and map the memory */
395	r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
396			     &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
397	if (r != 0) {
398		aprint_error_dev(sc->sc_dev,
399				 "virtqueue %d for %s allocation failed, "
400				 "error code %d\n", index, name, r);
401		goto err;
402	}
403	r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize,
404			   &vq->vq_vaddr, BUS_DMA_NOWAIT);
405	if (r != 0) {
406		aprint_error_dev(sc->sc_dev,
407				 "virtqueue %d for %s map failed, "
408				 "error code %d\n", index, name, r);
409		goto err;
410	}
411	r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
412			      BUS_DMA_NOWAIT, &vq->vq_dmamap);
413	if (r != 0) {
414		aprint_error_dev(sc->sc_dev,
415				 "virtqueue %d for %s dmamap creation failed, "
416				 "error code %d\n", index, name, r);
417		goto err;
418	}
419	r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
420			    vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT);
421	if (r != 0) {
422		aprint_error_dev(sc->sc_dev,
423				 "virtqueue %d for %s dmamap load failed, "
424				 "error code %d\n", index, name, r);
425		goto err;
426	}
427
428	/* set the vq address */
429	sc->sc_ops->setup_queue(sc, index,
430	    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
431
432	/* remember addresses and offsets for later use */
433	vq->vq_owner = sc;
434	vq->vq_num = vq_size;
435	vq->vq_index = index;
436	vq->vq_desc = vq->vq_vaddr;
437	vq->vq_availoffset = sizeof(struct vring_desc)*vq_size;
438	vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset);
439	vq->vq_usedoffset = allocsize1;
440	vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset);
441	if (allocsize3 > 0) {
442		vq->vq_indirectoffset = allocsize1 + allocsize2;
443		vq->vq_indirect = (void*)(((char*)vq->vq_desc)
444					  + vq->vq_indirectoffset);
445	}
446	vq->vq_bytesize = allocsize;
447	vq->vq_maxsegsize = maxsegsize;
448	vq->vq_maxnsegs = maxnsegs;
449
450	/* free slot management */
451	vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size,
452				     KM_SLEEP);
453	virtio_init_vq(sc, vq, false);
454
455	aprint_verbose_dev(sc->sc_dev,
456			   "allocated %u byte for virtqueue %d for %s, "
457			   "size %d\n", allocsize, index, name, vq_size);
458	if (allocsize3 > 0)
459		aprint_verbose_dev(sc->sc_dev,
460				   "using %d byte (%d entries) "
461				   "indirect descriptors\n",
462				   allocsize3, maxnsegs * vq_size);
463
464	sc->sc_nvqs++;
465
466	return 0;
467
468err:
469	sc->sc_ops->setup_queue(sc, index, 0);
470	if (vq->vq_dmamap)
471		bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
472	if (vq->vq_vaddr)
473		bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
474	if (vq->vq_segs[0].ds_addr)
475		bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
476	memset(vq, 0, sizeof(*vq));
477
478	return -1;
479}
480
481int
482virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
483{
484	struct vq_entry *qe;
485	int i = 0;
486
487	/* device must be already deactivated */
488	/* confirm the vq is empty */
489	SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) {
490		i++;
491	}
492	if (i != vq->vq_num) {
493		printf("%s: freeing non-empty vq, index %d\n",
494		       device_xname(sc->sc_dev), vq->vq_index);
495		return EBUSY;
496	}
497
498	/* tell device that there's no virtqueue any longer */
499	sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
500
501	kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num);
502	bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
503	bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
504	bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
505	bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
506	mutex_destroy(&vq->vq_freelist_lock);
507	mutex_destroy(&vq->vq_uring_lock);
508	mutex_destroy(&vq->vq_aring_lock);
509	memset(vq, 0, sizeof(*vq));
510
511	sc->sc_nvqs--;
512
513	return 0;
514}
515
516/*
517 * Free descriptor management.
518 */
519static struct vq_entry *
520vq_alloc_entry(struct virtqueue *vq)
521{
522	struct vq_entry *qe;
523
524	mutex_enter(&vq->vq_freelist_lock);
525	if (SIMPLEQ_EMPTY(&vq->vq_freelist)) {
526		mutex_exit(&vq->vq_freelist_lock);
527		return NULL;
528	}
529	qe = SIMPLEQ_FIRST(&vq->vq_freelist);
530	SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list);
531	mutex_exit(&vq->vq_freelist_lock);
532
533	return qe;
534}
535
536static void
537vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
538{
539	mutex_enter(&vq->vq_freelist_lock);
540	SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list);
541	mutex_exit(&vq->vq_freelist_lock);
542
543	return;
544}
545
546/*
547 * Enqueue several dmamaps as a single request.
548 */
549/*
550 * Typical usage:
551 *  <queue size> number of followings are stored in arrays
552 *  - command blocks (in dmamem) should be pre-allocated and mapped
553 *  - dmamaps for command blocks should be pre-allocated and loaded
554 *  - dmamaps for payload should be pre-allocated
555 *      r = virtio_enqueue_prep(sc, vq, &slot);		// allocate a slot
556 *	if (r)		// currently 0 or EAGAIN
557 *	  return r;
558 *	r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
559 *	if (r) {
560 *	  virtio_enqueue_abort(sc, vq, slot);
561 *	  return r;
562 *	}
563 *	r = virtio_enqueue_reserve(sc, vq, slot,
564 *				   dmamap_payload[slot]->dm_nsegs+1);
565 *							// ^ +1 for command
566 *	if (r) {	// currently 0 or EAGAIN
567 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
568 *	  return r;					// do not call abort()
569 *	}
570 *	<setup and prepare commands>
571 *	bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
572 *	bus_dmamap_sync(dmat, dmamap_payload[slot],...);
573 *	virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
574 *	virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
575 *	virtio_enqueue_commit(sc, vq, slot, true);
576 */
577
578/*
579 * enqueue_prep: allocate a slot number
580 */
581int
582virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
583{
584	struct vq_entry *qe1;
585
586	KASSERT(slotp != NULL);
587
588	qe1 = vq_alloc_entry(vq);
589	if (qe1 == NULL)
590		return EAGAIN;
591	/* next slot is not allocated yet */
592	qe1->qe_next = -1;
593	*slotp = qe1->qe_index;
594
595	return 0;
596}
597
598/*
599 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
600 */
601int
602virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
603		       int slot, int nsegs)
604{
605	int indirect;
606	struct vq_entry *qe1 = &vq->vq_entries[slot];
607
608	KASSERT(qe1->qe_next == -1);
609	KASSERT(1 <= nsegs && nsegs <= vq->vq_num);
610
611	if ((vq->vq_indirect != NULL) &&
612	    (nsegs >= MINSEG_INDIRECT) &&
613	    (nsegs <= vq->vq_maxnsegs))
614		indirect = 1;
615	else
616		indirect = 0;
617	qe1->qe_indirect = indirect;
618
619	if (indirect) {
620		struct vring_desc *vd;
621		int i;
622
623		vd = &vq->vq_desc[qe1->qe_index];
624		vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr
625			+ vq->vq_indirectoffset;
626		vd->addr += sizeof(struct vring_desc)
627			* vq->vq_maxnsegs * qe1->qe_index;
628		vd->len = sizeof(struct vring_desc) * nsegs;
629		vd->flags = VRING_DESC_F_INDIRECT;
630
631		vd = vq->vq_indirect;
632		vd += vq->vq_maxnsegs * qe1->qe_index;
633		qe1->qe_desc_base = vd;
634
635		for (i = 0; i < nsegs-1; i++) {
636			vd[i].flags = VRING_DESC_F_NEXT;
637		}
638		vd[i].flags = 0;
639		qe1->qe_next = 0;
640
641		return 0;
642	} else {
643		struct vring_desc *vd;
644		struct vq_entry *qe;
645		int i, s;
646
647		vd = &vq->vq_desc[0];
648		qe1->qe_desc_base = vd;
649		qe1->qe_next = qe1->qe_index;
650		s = slot;
651		for (i = 0; i < nsegs - 1; i++) {
652			qe = vq_alloc_entry(vq);
653			if (qe == NULL) {
654				vd[s].flags = 0;
655				virtio_enqueue_abort(sc, vq, slot);
656				return EAGAIN;
657			}
658			vd[s].flags = VRING_DESC_F_NEXT;
659			vd[s].next = qe->qe_index;
660			s = qe->qe_index;
661		}
662		vd[s].flags = 0;
663
664		return 0;
665	}
666}
667
668/*
669 * enqueue: enqueue a single dmamap.
670 */
671int
672virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
673	       bus_dmamap_t dmamap, bool write)
674{
675	struct vq_entry *qe1 = &vq->vq_entries[slot];
676	struct vring_desc *vd = qe1->qe_desc_base;
677	int i;
678	int s = qe1->qe_next;
679
680	KASSERT(s >= 0);
681	KASSERT(dmamap->dm_nsegs > 0);
682
683	for (i = 0; i < dmamap->dm_nsegs; i++) {
684		vd[s].addr = dmamap->dm_segs[i].ds_addr;
685		vd[s].len = dmamap->dm_segs[i].ds_len;
686		if (!write)
687			vd[s].flags |= VRING_DESC_F_WRITE;
688		s = vd[s].next;
689	}
690	qe1->qe_next = s;
691
692	return 0;
693}
694
695int
696virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
697		 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
698		 bool write)
699{
700	struct vq_entry *qe1 = &vq->vq_entries[slot];
701	struct vring_desc *vd = qe1->qe_desc_base;
702	int s = qe1->qe_next;
703
704	KASSERT(s >= 0);
705	KASSERT(dmamap->dm_nsegs == 1); /* XXX */
706	KASSERT((dmamap->dm_segs[0].ds_len > start) &&
707		(dmamap->dm_segs[0].ds_len >= start + len));
708
709	vd[s].addr = dmamap->dm_segs[0].ds_addr + start;
710	vd[s].len = len;
711	if (!write)
712		vd[s].flags |= VRING_DESC_F_WRITE;
713	qe1->qe_next = vd[s].next;
714
715	return 0;
716}
717
718/*
719 * enqueue_commit: add it to the aring.
720 */
721int
722virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
723		      bool notifynow)
724{
725	struct vq_entry *qe1;
726
727	if (slot < 0) {
728		mutex_enter(&vq->vq_aring_lock);
729		goto notify;
730	}
731	vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
732	qe1 = &vq->vq_entries[slot];
733	if (qe1->qe_indirect)
734		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
735	mutex_enter(&vq->vq_aring_lock);
736	vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot;
737
738notify:
739	if (notifynow) {
740		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
741		vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
742		membar_producer();
743		vq->vq_avail->idx = vq->vq_avail_idx;
744		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
745		membar_producer();
746		vq->vq_queued++;
747		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
748		membar_consumer();
749		if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY))
750			sc->sc_ops->kick(sc, vq->vq_index);
751	}
752	mutex_exit(&vq->vq_aring_lock);
753
754	return 0;
755}
756
757/*
758 * enqueue_abort: rollback.
759 */
760int
761virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
762{
763	struct vq_entry *qe = &vq->vq_entries[slot];
764	struct vring_desc *vd;
765	int s;
766
767	if (qe->qe_next < 0) {
768		vq_free_entry(vq, qe);
769		return 0;
770	}
771
772	s = slot;
773	vd = &vq->vq_desc[0];
774	while (vd[s].flags & VRING_DESC_F_NEXT) {
775		s = vd[s].next;
776		vq_free_entry(vq, qe);
777		qe = &vq->vq_entries[s];
778	}
779	vq_free_entry(vq, qe);
780	return 0;
781}
782
783/*
784 * Dequeue a request.
785 */
786/*
787 * dequeue: dequeue a request from uring; dmamap_sync for uring is
788 *	    already done in the interrupt handler.
789 */
790int
791virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
792	       int *slotp, int *lenp)
793{
794	uint16_t slot, usedidx;
795	struct vq_entry *qe;
796
797	if (vq->vq_used_idx == vq->vq_used->idx)
798		return ENOENT;
799	mutex_enter(&vq->vq_uring_lock);
800	usedidx = vq->vq_used_idx++;
801	mutex_exit(&vq->vq_uring_lock);
802	usedidx %= vq->vq_num;
803	slot = vq->vq_used->ring[usedidx].id;
804	qe = &vq->vq_entries[slot];
805
806	if (qe->qe_indirect)
807		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
808
809	if (slotp)
810		*slotp = slot;
811	if (lenp)
812		*lenp = vq->vq_used->ring[usedidx].len;
813
814	return 0;
815}
816
817/*
818 * dequeue_commit: complete dequeue; the slot is recycled for future use.
819 *                 if you forget to call this the slot will be leaked.
820 */
821int
822virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
823{
824	struct vq_entry *qe = &vq->vq_entries[slot];
825	struct vring_desc *vd = &vq->vq_desc[0];
826	int s = slot;
827
828	while (vd[s].flags & VRING_DESC_F_NEXT) {
829		s = vd[s].next;
830		vq_free_entry(vq, qe);
831		qe = &vq->vq_entries[s];
832	}
833	vq_free_entry(vq, qe);
834
835	return 0;
836}
837
838/*
839 * Attach a child, fill all the members.
840 */
841void
842virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl,
843		    struct virtqueue *vqs,
844		    virtio_callback config_change,
845		    virtio_callback intr_hand,
846		    int req_flags, int req_features, const char *feat_bits)
847{
848	char buf[256];
849	int features;
850
851	sc->sc_child = child;
852	sc->sc_ipl = ipl;
853	sc->sc_vqs = vqs;
854	sc->sc_config_change = config_change;
855	sc->sc_intrhand = intr_hand;
856	sc->sc_flags = req_flags;
857
858	features = virtio_negotiate_features(sc, req_features);
859	snprintb(buf, sizeof(buf), feat_bits, features);
860	aprint_normal(": Features: %s\n", buf);
861	aprint_naive("\n");
862}
863
864void
865virtio_child_attach_set_vqs(struct virtio_softc *sc,
866    struct virtqueue *vqs, int nvq_pairs)
867{
868
869	KASSERT(nvq_pairs == 1 ||
870	    (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) == 0);
871	if (nvq_pairs > 1)
872		sc->sc_child_mq = true;
873
874	sc->sc_vqs = vqs;
875}
876
877int
878virtio_child_attach_finish(struct virtio_softc *sc)
879{
880	int r;
881
882	r = sc->sc_ops->setup_interrupts(sc);
883	if (r != 0) {
884		aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n");
885		goto fail;
886	}
887
888	KASSERT(sc->sc_soft_ih == NULL);
889	if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) {
890		u_int flags = SOFTINT_NET;
891		if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
892			flags |= SOFTINT_MPSAFE;
893
894		sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc);
895		if (sc->sc_soft_ih == NULL) {
896			sc->sc_ops->free_interrupts(sc);
897			aprint_error_dev(sc->sc_dev,
898			    "failed to establish soft interrupt\n");
899			goto fail;
900		}
901	}
902
903	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
904	return 0;
905
906fail:
907	if (sc->sc_soft_ih) {
908		softint_disestablish(sc->sc_soft_ih);
909		sc->sc_soft_ih = NULL;
910	}
911
912	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
913	return 1;
914}
915
916void
917virtio_child_detach(struct virtio_softc *sc)
918{
919	sc->sc_child = NULL;
920	sc->sc_vqs = NULL;
921
922	virtio_device_reset(sc);
923
924	sc->sc_ops->free_interrupts(sc);
925
926	if (sc->sc_soft_ih) {
927		softint_disestablish(sc->sc_soft_ih);
928		sc->sc_soft_ih = NULL;
929	}
930}
931
932void
933virtio_child_attach_failed(struct virtio_softc *sc)
934{
935	virtio_child_detach(sc);
936
937	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
938
939	sc->sc_child = VIRTIO_CHILD_FAILED;
940}
941
942bus_dma_tag_t
943virtio_dmat(struct virtio_softc *sc)
944{
945	return sc->sc_dmat;
946}
947
948device_t
949virtio_child(struct virtio_softc *sc)
950{
951	return sc->sc_child;
952}
953
954int
955virtio_intrhand(struct virtio_softc *sc)
956{
957	return (sc->sc_intrhand)(sc);
958}
959
960uint32_t
961virtio_features(struct virtio_softc *sc)
962{
963	return sc->sc_features;
964}
965
966int
967virtiobusprint(void *aux, const char *pnp)
968{
969	struct virtio_attach_args * const va = aux;
970
971	if (va->sc_childdevid == 0)
972		return QUIET;	/* No device present */
973
974	if (pnp)
975		aprint_normal("Device ID %d at %s", va->sc_childdevid, pnp);
976
977	return UNCONF;
978}
979
980MODULE(MODULE_CLASS_DRIVER, virtio, NULL);
981
982#ifdef _MODULE
983#include "ioconf.c"
984#endif
985
986static int
987virtio_modcmd(modcmd_t cmd, void *opaque)
988{
989	int error = 0;
990
991#ifdef _MODULE
992	switch (cmd) {
993	case MODULE_CMD_INIT:
994		error = config_init_component(cfdriver_ioconf_virtio,
995		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
996		break;
997	case MODULE_CMD_FINI:
998		error = config_fini_component(cfdriver_ioconf_virtio,
999		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
1000		break;
1001	default:
1002		error = ENOTTY;
1003		break;
1004	}
1005#endif
1006
1007	return error;
1008}
1009