virtio.c revision 1.37 
1/*	$NetBSD: virtio.c,v 1.37 2019/01/14 14:55:37 yamaguchi Exp $	*/
2
3/*
4 * Copyright (c) 2010 Minoura Makoto.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.37 2019/01/14 14:55:37 yamaguchi Exp $");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/atomic.h>
35#include <sys/bus.h>
36#include <sys/device.h>
37#include <sys/kmem.h>
38#include <sys/module.h>
39
40#define VIRTIO_PRIVATE
41
42#include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
43#include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
44
45#define MINSEG_INDIRECT		2 /* use indirect if nsegs >= this value */
46
47static void	virtio_init_vq(struct virtio_softc *,
48		    struct virtqueue *, const bool);
49
50void
51virtio_set_status(struct virtio_softc *sc, int status)
52{
53	sc->sc_ops->set_status(sc, status);
54}
55
56/*
57 * Reset the device.
58 */
59/*
60 * To reset the device to a known state, do following:
61 *	virtio_reset(sc);	     // this will stop the device activity
62 *	<dequeue finished requests>; // virtio_dequeue() still can be called
63 *	<revoke pending requests in the vqs if any>;
64 *	virtio_reinit_begin(sc);     // dequeue prohibitted
65 *	newfeatures = virtio_negotiate_features(sc, requestedfeatures);
66 *	<some other initialization>;
67 *	virtio_reinit_end(sc);	     // device activated; enqueue allowed
68 * Once attached, feature negotiation can only be allowed after virtio_reset.
69 */
70void
71virtio_reset(struct virtio_softc *sc)
72{
73	virtio_device_reset(sc);
74}
75
76void
77virtio_reinit_start(struct virtio_softc *sc)
78{
79	int i;
80
81	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
82	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
83	for (i = 0; i < sc->sc_nvqs; i++) {
84		int n;
85		struct virtqueue *vq = &sc->sc_vqs[i];
86		n = sc->sc_ops->read_queue_size(sc, vq->vq_index);
87		if (n == 0)	/* vq disappeared */
88			continue;
89		if (n != vq->vq_num) {
90			panic("%s: virtqueue size changed, vq index %d\n",
91			      device_xname(sc->sc_dev),
92			      vq->vq_index);
93		}
94		virtio_init_vq(sc, vq, true);
95		sc->sc_ops->setup_queue(sc, vq->vq_index,
96		    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
97	}
98}
99
100void
101virtio_reinit_end(struct virtio_softc *sc)
102{
103	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
104}
105
106/*
107 * Feature negotiation.
108 */
109uint32_t
110virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
111{
112	uint32_t r;
113
114	if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
115	    !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
116		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
117	r = sc->sc_ops->neg_features(sc, guest_features);
118	sc->sc_features = r;
119	if (r & VIRTIO_F_RING_INDIRECT_DESC)
120		sc->sc_indirect = true;
121	else
122		sc->sc_indirect = false;
123
124	return r;
125}
126
127/*
128 * Device configuration registers.
129 */
130uint8_t
131virtio_read_device_config_1(struct virtio_softc *sc, int index)
132{
133	return sc->sc_ops->read_dev_cfg_1(sc, index);
134}
135
136uint16_t
137virtio_read_device_config_2(struct virtio_softc *sc, int index)
138{
139	return sc->sc_ops->read_dev_cfg_2(sc, index);
140}
141
142uint32_t
143virtio_read_device_config_4(struct virtio_softc *sc, int index)
144{
145	return sc->sc_ops->read_dev_cfg_4(sc, index);
146}
147
148uint64_t
149virtio_read_device_config_8(struct virtio_softc *sc, int index)
150{
151	return sc->sc_ops->read_dev_cfg_8(sc, index);
152}
153
154void
155virtio_write_device_config_1(struct virtio_softc *sc,
156			     int index, uint8_t value)
157{
158	return sc->sc_ops->write_dev_cfg_1(sc, index, value);
159}
160
161void
162virtio_write_device_config_2(struct virtio_softc *sc,
163			     int index, uint16_t value)
164{
165	return sc->sc_ops->write_dev_cfg_2(sc, index, value);
166}
167
168void
169virtio_write_device_config_4(struct virtio_softc *sc,
170			     int index, uint32_t value)
171{
172	return sc->sc_ops->write_dev_cfg_4(sc, index, value);
173}
174
175void
176virtio_write_device_config_8(struct virtio_softc *sc,
177			     int index, uint64_t value)
178{
179	return sc->sc_ops->write_dev_cfg_8(sc, index, value);
180}
181
182/*
183 * Interrupt handler.
184 */
185static void
186virtio_soft_intr(void *arg)
187{
188	struct virtio_softc *sc = arg;
189
190	KASSERT(sc->sc_intrhand != NULL);
191
192	(sc->sc_intrhand)(sc);
193}
194
195/*
196 * dmamap sync operations for a virtqueue.
197 */
198static inline void
199vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
200{
201	/* availoffset == sizeof(vring_desc)*vq_num */
202	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
203			ops);
204}
205
206static inline void
207vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
208{
209	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
210			vq->vq_availoffset,
211			offsetof(struct vring_avail, ring)
212			 + vq->vq_num * sizeof(uint16_t),
213			ops);
214}
215
216static inline void
217vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops)
218{
219	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
220			vq->vq_usedoffset,
221			offsetof(struct vring_used, ring)
222			 + vq->vq_num * sizeof(struct vring_used_elem),
223			ops);
224}
225
226static inline void
227vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
228		     int ops)
229{
230	int offset = vq->vq_indirectoffset
231		      + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
232
233	bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
234			offset, sizeof(struct vring_desc) * vq->vq_maxnsegs,
235			ops);
236}
237
238static void
239virtio_vq_soft_intr(void *arg)
240{
241	struct virtqueue *vq = arg;
242
243	KASSERT(vq->vq_intrhand != NULL);
244
245	(vq->vq_intrhand)(vq);
246}
247
248static int
249virtio_vq_softint_establish(struct virtio_softc *sc)
250{
251	struct virtqueue *vq;
252	int qid;
253	u_int flags;
254
255	flags = SOFTINT_NET;
256	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
257		flags |= SOFTINT_MPSAFE;
258
259	for (qid = 0; qid < sc->sc_nvqs; qid++) {
260		vq = &sc->sc_vqs[qid];
261		vq->vq_soft_ih =
262		    softint_establish(flags, virtio_vq_soft_intr, vq);
263		if (vq->vq_soft_ih == NULL)
264			return -1;
265	}
266
267	return 0;
268}
269
270static void
271virtio_vq_softint_disestablish(struct virtio_softc *sc)
272{
273	struct virtqueue *vq;
274	int qid;
275
276	for (qid = 0; qid < sc->sc_nvqs; qid++) {
277		vq = &sc->sc_vqs[qid];
278		if (vq->vq_soft_ih == NULL)
279			continue;
280
281		softint_disestablish(vq->vq_soft_ih);
282		vq->vq_soft_ih = NULL;
283	}
284}
285
286/*
287 * Can be used as sc_intrhand.
288 */
289/*
290 * Scan vq, bus_dmamap_sync for the vqs (not for the payload),
291 * and calls (*vq_done)() if some entries are consumed.
292 */
293static int
294virtio_vq_intr_common(struct virtqueue *vq)
295{
296	struct virtio_softc *sc = vq->vq_owner;
297	int r = 0;
298
299	if (vq->vq_queued) {
300		vq->vq_queued = 0;
301		vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
302	}
303	vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
304	membar_consumer();
305	if (vq->vq_used_idx != vq->vq_used->idx) {
306		if (vq->vq_done)
307			r |= (vq->vq_done)(vq);
308	}
309
310	return r;
311}
312
313int
314virtio_vq_intr(struct virtio_softc *sc)
315{
316	struct virtqueue *vq;
317	int i, r = 0;
318
319	for (i = 0; i < sc->sc_nvqs; i++) {
320		vq = &sc->sc_vqs[i];
321		r |= virtio_vq_intr_common(vq);
322	}
323
324	return r;
325}
326
327static int
328virtio_vq_mq_intr(struct virtqueue *vq)
329{
330
331	return virtio_vq_intr_common(vq);
332}
333
334/*
335 * Start/stop vq interrupt.  No guarantee.
336 */
337void
338virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
339{
340	vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
341	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
342	vq->vq_queued++;
343}
344
345void
346virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
347{
348	vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
349	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
350	vq->vq_queued++;
351}
352
353/*
354 * Initialize vq structure.
355 */
356static void
357virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq,
358    const bool reinit)
359{
360	int i, j;
361	int vq_size = vq->vq_num;
362
363	memset(vq->vq_vaddr, 0, vq->vq_bytesize);
364
365	/* build the indirect descriptor chain */
366	if (vq->vq_indirect != NULL) {
367		struct vring_desc *vd;
368
369		for (i = 0; i < vq_size; i++) {
370			vd = vq->vq_indirect;
371			vd += vq->vq_maxnsegs * i;
372			for (j = 0; j < vq->vq_maxnsegs-1; j++) {
373				vd[j].next = j + 1;
374			}
375		}
376	}
377
378	/* free slot management */
379	SIMPLEQ_INIT(&vq->vq_freelist);
380	for (i = 0; i < vq_size; i++) {
381		SIMPLEQ_INSERT_TAIL(&vq->vq_freelist,
382				    &vq->vq_entries[i], qe_list);
383		vq->vq_entries[i].qe_index = i;
384	}
385	if (!reinit)
386		mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl);
387
388	/* enqueue/dequeue status */
389	vq->vq_avail_idx = 0;
390	vq->vq_used_idx = 0;
391	vq->vq_queued = 0;
392	if (!reinit) {
393		mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
394		mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
395	}
396	vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
397	vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
398	vq->vq_queued++;
399}
400
401/*
402 * Allocate/free a vq.
403 */
404int
405virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index,
406    int maxsegsize, int maxnsegs, const char *name)
407{
408	int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0;
409	int rsegs, r;
410#define VIRTQUEUE_ALIGN(n)	(((n)+(VIRTIO_PAGE_SIZE-1))&	\
411				 ~(VIRTIO_PAGE_SIZE-1))
412
413	/* Make sure callers allocate vqs in order */
414	KASSERT(sc->sc_nvqs == index);
415
416	memset(vq, 0, sizeof(*vq));
417
418	vq_size = sc->sc_ops->read_queue_size(sc, index);
419	if (vq_size == 0) {
420		aprint_error_dev(sc->sc_dev,
421				 "virtqueue not exist, index %d for %s\n",
422				 index, name);
423		goto err;
424	}
425	/* allocsize1: descriptor table + avail ring + pad */
426	allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size
427				     + sizeof(uint16_t)*(2+vq_size));
428	/* allocsize2: used ring + pad */
429	allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2
430				     + sizeof(struct vring_used_elem)*vq_size);
431	/* allocsize3: indirect table */
432	if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT)
433		allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size;
434	else
435		allocsize3 = 0;
436	allocsize = allocsize1 + allocsize2 + allocsize3;
437
438	/* alloc and map the memory */
439	r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
440			     &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
441	if (r != 0) {
442		aprint_error_dev(sc->sc_dev,
443				 "virtqueue %d for %s allocation failed, "
444				 "error code %d\n", index, name, r);
445		goto err;
446	}
447	r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize,
448			   &vq->vq_vaddr, BUS_DMA_NOWAIT);
449	if (r != 0) {
450		aprint_error_dev(sc->sc_dev,
451				 "virtqueue %d for %s map failed, "
452				 "error code %d\n", index, name, r);
453		goto err;
454	}
455	r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
456			      BUS_DMA_NOWAIT, &vq->vq_dmamap);
457	if (r != 0) {
458		aprint_error_dev(sc->sc_dev,
459				 "virtqueue %d for %s dmamap creation failed, "
460				 "error code %d\n", index, name, r);
461		goto err;
462	}
463	r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
464			    vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT);
465	if (r != 0) {
466		aprint_error_dev(sc->sc_dev,
467				 "virtqueue %d for %s dmamap load failed, "
468				 "error code %d\n", index, name, r);
469		goto err;
470	}
471
472	/* set the vq address */
473	sc->sc_ops->setup_queue(sc, index,
474	    vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE);
475
476	/* remember addresses and offsets for later use */
477	vq->vq_owner = sc;
478	vq->vq_intrhand = virtio_vq_mq_intr;
479	vq->vq_num = vq_size;
480	vq->vq_index = index;
481	vq->vq_desc = vq->vq_vaddr;
482	vq->vq_availoffset = sizeof(struct vring_desc)*vq_size;
483	vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset);
484	vq->vq_usedoffset = allocsize1;
485	vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset);
486	if (allocsize3 > 0) {
487		vq->vq_indirectoffset = allocsize1 + allocsize2;
488		vq->vq_indirect = (void*)(((char*)vq->vq_desc)
489					  + vq->vq_indirectoffset);
490	}
491	vq->vq_bytesize = allocsize;
492	vq->vq_maxsegsize = maxsegsize;
493	vq->vq_maxnsegs = maxnsegs;
494
495	/* free slot management */
496	vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size,
497				     KM_NOSLEEP);
498	if (vq->vq_entries == NULL) {
499		r = ENOMEM;
500		goto err;
501	}
502
503	virtio_init_vq(sc, vq, false);
504
505	aprint_verbose_dev(sc->sc_dev,
506			   "allocated %u byte for virtqueue %d for %s, "
507			   "size %d\n", allocsize, index, name, vq_size);
508	if (allocsize3 > 0)
509		aprint_verbose_dev(sc->sc_dev,
510				   "using %d byte (%d entries) "
511				   "indirect descriptors\n",
512				   allocsize3, maxnsegs * vq_size);
513
514	sc->sc_nvqs++;
515
516	return 0;
517
518err:
519	sc->sc_ops->setup_queue(sc, index, 0);
520	if (vq->vq_dmamap)
521		bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
522	if (vq->vq_vaddr)
523		bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
524	if (vq->vq_segs[0].ds_addr)
525		bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
526	memset(vq, 0, sizeof(*vq));
527
528	return -1;
529}
530
531int
532virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
533{
534	struct vq_entry *qe;
535	int i = 0;
536
537	/* device must be already deactivated */
538	/* confirm the vq is empty */
539	SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) {
540		i++;
541	}
542	if (i != vq->vq_num) {
543		printf("%s: freeing non-empty vq, index %d\n",
544		       device_xname(sc->sc_dev), vq->vq_index);
545		return EBUSY;
546	}
547
548	/* tell device that there's no virtqueue any longer */
549	sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
550
551	kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num);
552	bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
553	bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
554	bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
555	bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
556	mutex_destroy(&vq->vq_freelist_lock);
557	mutex_destroy(&vq->vq_uring_lock);
558	mutex_destroy(&vq->vq_aring_lock);
559	memset(vq, 0, sizeof(*vq));
560
561	sc->sc_nvqs--;
562
563	return 0;
564}
565
566/*
567 * Free descriptor management.
568 */
569static struct vq_entry *
570vq_alloc_entry(struct virtqueue *vq)
571{
572	struct vq_entry *qe;
573
574	mutex_enter(&vq->vq_freelist_lock);
575	if (SIMPLEQ_EMPTY(&vq->vq_freelist)) {
576		mutex_exit(&vq->vq_freelist_lock);
577		return NULL;
578	}
579	qe = SIMPLEQ_FIRST(&vq->vq_freelist);
580	SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list);
581	mutex_exit(&vq->vq_freelist_lock);
582
583	return qe;
584}
585
586static void
587vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
588{
589	mutex_enter(&vq->vq_freelist_lock);
590	SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list);
591	mutex_exit(&vq->vq_freelist_lock);
592
593	return;
594}
595
596/*
597 * Enqueue several dmamaps as a single request.
598 */
599/*
600 * Typical usage:
601 *  <queue size> number of followings are stored in arrays
602 *  - command blocks (in dmamem) should be pre-allocated and mapped
603 *  - dmamaps for command blocks should be pre-allocated and loaded
604 *  - dmamaps for payload should be pre-allocated
605 *      r = virtio_enqueue_prep(sc, vq, &slot);		// allocate a slot
606 *	if (r)		// currently 0 or EAGAIN
607 *	  return r;
608 *	r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
609 *	if (r) {
610 *	  virtio_enqueue_abort(sc, vq, slot);
611 *	  return r;
612 *	}
613 *	r = virtio_enqueue_reserve(sc, vq, slot,
614 *				   dmamap_payload[slot]->dm_nsegs+1);
615 *							// ^ +1 for command
616 *	if (r) {	// currently 0 or EAGAIN
617 *	  bus_dmamap_unload(dmat, dmamap_payload[slot]);
618 *	  return r;					// do not call abort()
619 *	}
620 *	<setup and prepare commands>
621 *	bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
622 *	bus_dmamap_sync(dmat, dmamap_payload[slot],...);
623 *	virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
624 *	virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
625 *	virtio_enqueue_commit(sc, vq, slot, true);
626 */
627
628/*
629 * enqueue_prep: allocate a slot number
630 */
631int
632virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
633{
634	struct vq_entry *qe1;
635
636	KASSERT(slotp != NULL);
637
638	qe1 = vq_alloc_entry(vq);
639	if (qe1 == NULL)
640		return EAGAIN;
641	/* next slot is not allocated yet */
642	qe1->qe_next = -1;
643	*slotp = qe1->qe_index;
644
645	return 0;
646}
647
648/*
649 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
650 */
651int
652virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
653		       int slot, int nsegs)
654{
655	int indirect;
656	struct vq_entry *qe1 = &vq->vq_entries[slot];
657
658	KASSERT(qe1->qe_next == -1);
659	KASSERT(1 <= nsegs && nsegs <= vq->vq_num);
660
661	if ((vq->vq_indirect != NULL) &&
662	    (nsegs >= MINSEG_INDIRECT) &&
663	    (nsegs <= vq->vq_maxnsegs))
664		indirect = 1;
665	else
666		indirect = 0;
667	qe1->qe_indirect = indirect;
668
669	if (indirect) {
670		struct vring_desc *vd;
671		int i;
672
673		vd = &vq->vq_desc[qe1->qe_index];
674		vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr
675			+ vq->vq_indirectoffset;
676		vd->addr += sizeof(struct vring_desc)
677			* vq->vq_maxnsegs * qe1->qe_index;
678		vd->len = sizeof(struct vring_desc) * nsegs;
679		vd->flags = VRING_DESC_F_INDIRECT;
680
681		vd = vq->vq_indirect;
682		vd += vq->vq_maxnsegs * qe1->qe_index;
683		qe1->qe_desc_base = vd;
684
685		for (i = 0; i < nsegs-1; i++) {
686			vd[i].flags = VRING_DESC_F_NEXT;
687		}
688		vd[i].flags = 0;
689		qe1->qe_next = 0;
690
691		return 0;
692	} else {
693		struct vring_desc *vd;
694		struct vq_entry *qe;
695		int i, s;
696
697		vd = &vq->vq_desc[0];
698		qe1->qe_desc_base = vd;
699		qe1->qe_next = qe1->qe_index;
700		s = slot;
701		for (i = 0; i < nsegs - 1; i++) {
702			qe = vq_alloc_entry(vq);
703			if (qe == NULL) {
704				vd[s].flags = 0;
705				virtio_enqueue_abort(sc, vq, slot);
706				return EAGAIN;
707			}
708			vd[s].flags = VRING_DESC_F_NEXT;
709			vd[s].next = qe->qe_index;
710			s = qe->qe_index;
711		}
712		vd[s].flags = 0;
713
714		return 0;
715	}
716}
717
718/*
719 * enqueue: enqueue a single dmamap.
720 */
721int
722virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
723	       bus_dmamap_t dmamap, bool write)
724{
725	struct vq_entry *qe1 = &vq->vq_entries[slot];
726	struct vring_desc *vd = qe1->qe_desc_base;
727	int i;
728	int s = qe1->qe_next;
729
730	KASSERT(s >= 0);
731	KASSERT(dmamap->dm_nsegs > 0);
732
733	for (i = 0; i < dmamap->dm_nsegs; i++) {
734		vd[s].addr = dmamap->dm_segs[i].ds_addr;
735		vd[s].len = dmamap->dm_segs[i].ds_len;
736		if (!write)
737			vd[s].flags |= VRING_DESC_F_WRITE;
738		s = vd[s].next;
739	}
740	qe1->qe_next = s;
741
742	return 0;
743}
744
745int
746virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
747		 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
748		 bool write)
749{
750	struct vq_entry *qe1 = &vq->vq_entries[slot];
751	struct vring_desc *vd = qe1->qe_desc_base;
752	int s = qe1->qe_next;
753
754	KASSERT(s >= 0);
755	KASSERT(dmamap->dm_nsegs == 1); /* XXX */
756	KASSERT((dmamap->dm_segs[0].ds_len > start) &&
757		(dmamap->dm_segs[0].ds_len >= start + len));
758
759	vd[s].addr = dmamap->dm_segs[0].ds_addr + start;
760	vd[s].len = len;
761	if (!write)
762		vd[s].flags |= VRING_DESC_F_WRITE;
763	qe1->qe_next = vd[s].next;
764
765	return 0;
766}
767
768/*
769 * enqueue_commit: add it to the aring.
770 */
771int
772virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
773		      bool notifynow)
774{
775	struct vq_entry *qe1;
776
777	if (slot < 0) {
778		mutex_enter(&vq->vq_aring_lock);
779		goto notify;
780	}
781	vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
782	qe1 = &vq->vq_entries[slot];
783	if (qe1->qe_indirect)
784		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
785	mutex_enter(&vq->vq_aring_lock);
786	vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot;
787
788notify:
789	if (notifynow) {
790		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
791		vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD);
792		membar_producer();
793		vq->vq_avail->idx = vq->vq_avail_idx;
794		vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE);
795		membar_producer();
796		vq->vq_queued++;
797		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
798		membar_consumer();
799		if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY))
800			sc->sc_ops->kick(sc, vq->vq_index);
801	}
802	mutex_exit(&vq->vq_aring_lock);
803
804	return 0;
805}
806
807/*
808 * enqueue_abort: rollback.
809 */
810int
811virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
812{
813	struct vq_entry *qe = &vq->vq_entries[slot];
814	struct vring_desc *vd;
815	int s;
816
817	if (qe->qe_next < 0) {
818		vq_free_entry(vq, qe);
819		return 0;
820	}
821
822	s = slot;
823	vd = &vq->vq_desc[0];
824	while (vd[s].flags & VRING_DESC_F_NEXT) {
825		s = vd[s].next;
826		vq_free_entry(vq, qe);
827		qe = &vq->vq_entries[s];
828	}
829	vq_free_entry(vq, qe);
830	return 0;
831}
832
833/*
834 * Dequeue a request.
835 */
836/*
837 * dequeue: dequeue a request from uring; dmamap_sync for uring is
838 *	    already done in the interrupt handler.
839 */
840int
841virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
842	       int *slotp, int *lenp)
843{
844	uint16_t slot, usedidx;
845	struct vq_entry *qe;
846
847	if (vq->vq_used_idx == vq->vq_used->idx)
848		return ENOENT;
849	mutex_enter(&vq->vq_uring_lock);
850	usedidx = vq->vq_used_idx++;
851	mutex_exit(&vq->vq_uring_lock);
852	usedidx %= vq->vq_num;
853	slot = vq->vq_used->ring[usedidx].id;
854	qe = &vq->vq_entries[slot];
855
856	if (qe->qe_indirect)
857		vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
858
859	if (slotp)
860		*slotp = slot;
861	if (lenp)
862		*lenp = vq->vq_used->ring[usedidx].len;
863
864	return 0;
865}
866
867/*
868 * dequeue_commit: complete dequeue; the slot is recycled for future use.
869 *                 if you forget to call this the slot will be leaked.
870 */
871int
872virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
873{
874	struct vq_entry *qe = &vq->vq_entries[slot];
875	struct vring_desc *vd = &vq->vq_desc[0];
876	int s = slot;
877
878	while (vd[s].flags & VRING_DESC_F_NEXT) {
879		s = vd[s].next;
880		vq_free_entry(vq, qe);
881		qe = &vq->vq_entries[s];
882	}
883	vq_free_entry(vq, qe);
884
885	return 0;
886}
887
888/*
889 * Attach a child, fill all the members.
890 */
891void
892virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl,
893		    struct virtqueue *vqs,
894		    virtio_callback config_change,
895		    virtio_callback intr_hand,
896		    int req_flags, int req_features, const char *feat_bits)
897{
898	char buf[256];
899	int features;
900
901	sc->sc_child = child;
902	sc->sc_ipl = ipl;
903	sc->sc_vqs = vqs;
904	sc->sc_config_change = config_change;
905	sc->sc_intrhand = intr_hand;
906	sc->sc_flags = req_flags;
907
908	features = virtio_negotiate_features(sc, req_features);
909	snprintb(buf, sizeof(buf), feat_bits, features);
910	aprint_normal(": Features: %s\n", buf);
911	aprint_naive("\n");
912}
913
914void
915virtio_child_attach_set_vqs(struct virtio_softc *sc,
916    struct virtqueue *vqs, int nvq_pairs)
917{
918	if (nvq_pairs > 1)
919		sc->sc_child_mq = true;
920
921	sc->sc_vqs = vqs;
922}
923
924int
925virtio_child_attach_finish(struct virtio_softc *sc)
926{
927	int r;
928
929	r = sc->sc_ops->setup_interrupts(sc);
930	if (r != 0) {
931		aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n");
932		goto fail;
933	}
934
935	KASSERT(sc->sc_soft_ih == NULL);
936	if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) {
937		u_int flags = SOFTINT_NET;
938		if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
939			flags |= SOFTINT_MPSAFE;
940
941		sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc);
942		if (sc->sc_soft_ih == NULL) {
943			sc->sc_ops->free_interrupts(sc);
944			aprint_error_dev(sc->sc_dev,
945			    "failed to establish soft interrupt\n");
946			goto fail;
947		}
948
949		if (sc->sc_child_mq) {
950			r = virtio_vq_softint_establish(sc);
951			aprint_error_dev(sc->sc_dev,
952			    "failed to establish softint interrupt\n");
953			goto fail;
954		}
955	}
956
957	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
958	return 0;
959
960fail:
961	if (sc->sc_soft_ih) {
962		softint_disestablish(sc->sc_soft_ih);
963		sc->sc_soft_ih = NULL;
964	}
965
966	virtio_vq_softint_disestablish(sc);
967
968	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
969	return 1;
970}
971
972void
973virtio_child_detach(struct virtio_softc *sc)
974{
975	sc->sc_child = NULL;
976	sc->sc_vqs = NULL;
977
978	virtio_device_reset(sc);
979
980	sc->sc_ops->free_interrupts(sc);
981
982	if (sc->sc_soft_ih) {
983		softint_disestablish(sc->sc_soft_ih);
984		sc->sc_soft_ih = NULL;
985	}
986}
987
988void
989virtio_child_attach_failed(struct virtio_softc *sc)
990{
991	virtio_child_detach(sc);
992
993	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
994
995	sc->sc_child = VIRTIO_CHILD_FAILED;
996}
997
998bus_dma_tag_t
999virtio_dmat(struct virtio_softc *sc)
1000{
1001	return sc->sc_dmat;
1002}
1003
1004device_t
1005virtio_child(struct virtio_softc *sc)
1006{
1007	return sc->sc_child;
1008}
1009
1010int
1011virtio_intrhand(struct virtio_softc *sc)
1012{
1013	return (sc->sc_intrhand)(sc);
1014}
1015
1016uint32_t
1017virtio_features(struct virtio_softc *sc)
1018{
1019	return sc->sc_features;
1020}
1021
1022int
1023virtiobusprint(void *aux, const char *pnp)
1024{
1025	struct virtio_attach_args * const va = aux;
1026
1027	if (va->sc_childdevid == 0)
1028		return QUIET;	/* No device present */
1029
1030	if (pnp)
1031		aprint_normal("Device ID %d at %s", va->sc_childdevid, pnp);
1032
1033	return UNCONF;
1034}
1035
1036MODULE(MODULE_CLASS_DRIVER, virtio, NULL);
1037
1038#ifdef _MODULE
1039#include "ioconf.c"
1040#endif
1041
1042static int
1043virtio_modcmd(modcmd_t cmd, void *opaque)
1044{
1045	int error = 0;
1046
1047#ifdef _MODULE
1048	switch (cmd) {
1049	case MODULE_CMD_INIT:
1050		error = config_init_component(cfdriver_ioconf_virtio,
1051		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
1052		break;
1053	case MODULE_CMD_FINI:
1054		error = config_fini_component(cfdriver_ioconf_virtio,
1055		    cfattach_ioconf_virtio, cfdata_ioconf_virtio);
1056		break;
1057	default:
1058		error = ENOTTY;
1059		break;
1060	}
1061#endif
1062
1063	return error;
1064}
1065