Deleted Added
sdiff udiff text old ( 227652 ) new ( 234270 )
full compact
virtqueue.c (227652) virtqueue.c (234270)
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * Implements the virtqueue interface as basically described
29 * in the original VirtIO paper.
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * Implements the virtqueue interface as basically described
29 * in the original VirtIO paper.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 227652 2011-11-18 05:43:43Z grehan $");
33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 234270 2012-04-14 05:48:04Z grehan $");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/sglist.h>
40#include <vm/vm.h>
41#include <vm/pmap.h>

--- 13 unchanged lines hidden (view full) ---

55
56struct virtqueue {
57 device_t vq_dev;
58 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
59 uint16_t vq_queue_index;
60 uint16_t vq_nentries;
61 uint32_t vq_flags;
62#define VIRTQUEUE_FLAG_INDIRECT 0x0001
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/sglist.h>
40#include <vm/vm.h>
41#include <vm/pmap.h>

--- 13 unchanged lines hidden (view full) ---

55
56struct virtqueue {
57 device_t vq_dev;
58 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
59 uint16_t vq_queue_index;
60 uint16_t vq_nentries;
61 uint32_t vq_flags;
62#define VIRTQUEUE_FLAG_INDIRECT 0x0001
63#define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
63
64 int vq_alignment;
65 int vq_ring_size;
66 void *vq_ring_mem;
67 int vq_max_indirect_size;
68 int vq_indirect_mem_size;
69 virtqueue_intr_t *vq_intrhand;
70 void *vq_intrhand_arg;

--- 50 unchanged lines hidden (view full) ---

121
122static void vq_ring_init(struct virtqueue *);
123static void vq_ring_update_avail(struct virtqueue *, uint16_t);
124static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
125 struct vring_desc *, uint16_t, struct sglist *, int, int);
126static int vq_ring_use_indirect(struct virtqueue *, int);
127static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
128 struct sglist *, int, int);
64
65 int vq_alignment;
66 int vq_ring_size;
67 void *vq_ring_mem;
68 int vq_max_indirect_size;
69 int vq_indirect_mem_size;
70 virtqueue_intr_t *vq_intrhand;
71 void *vq_intrhand_arg;

--- 50 unchanged lines hidden (view full) ---

122
123static void vq_ring_init(struct virtqueue *);
124static void vq_ring_update_avail(struct virtqueue *, uint16_t);
125static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
126 struct vring_desc *, uint16_t, struct sglist *, int, int);
127static int vq_ring_use_indirect(struct virtqueue *, int);
128static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
129 struct sglist *, int, int);
129static void vq_ring_notify_host(struct virtqueue *, int);
130static int vq_ring_must_notify_host(struct virtqueue *);
131static void vq_ring_notify_host(struct virtqueue *);
130static void vq_ring_free_chain(struct virtqueue *, uint16_t);
131
132uint64_t
133virtqueue_filter_features(uint64_t features)
134{
135 uint64_t mask;
136
137 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
138 mask |= VIRTIO_RING_F_INDIRECT_DESC;
132static void vq_ring_free_chain(struct virtqueue *, uint16_t);
133
134uint64_t
135virtqueue_filter_features(uint64_t features)
136{
137 uint64_t mask;
138
139 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
140 mask |= VIRTIO_RING_F_INDIRECT_DESC;
141 mask |= VIRTIO_RING_F_EVENT_IDX;
139
140 return (features & mask);
141}
142
143int
144virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
145 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
146{

--- 32 unchanged lines hidden (view full) ---

179 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
180 vq->vq_queue_index = queue;
181 vq->vq_alignment = align;
182 vq->vq_nentries = size;
183 vq->vq_free_cnt = size;
184 vq->vq_intrhand = info->vqai_intr;
185 vq->vq_intrhand_arg = info->vqai_intr_arg;
186
142
143 return (features & mask);
144}
145
146int
147virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
148 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
149{

--- 32 unchanged lines hidden (view full) ---

182 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
183 vq->vq_queue_index = queue;
184 vq->vq_alignment = align;
185 vq->vq_nentries = size;
186 vq->vq_free_cnt = size;
187 vq->vq_intrhand = info->vqai_intr;
188 vq->vq_intrhand_arg = info->vqai_intr_arg;
189
190 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
191 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
192
187 if (info->vqai_maxindirsz > 1) {
188 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
189 if (error)
190 goto fail;
191 }
192
193 vq->vq_ring_size = round_page(vring_size(size, align));
194 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,

--- 184 unchanged lines hidden (view full) ---

379{
380
381 return (vq->vq_free_cnt == 0);
382}
383
384void
385virtqueue_notify(struct virtqueue *vq)
386{
193 if (info->vqai_maxindirsz > 1) {
194 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
195 if (error)
196 goto fail;
197 }
198
199 vq->vq_ring_size = round_page(vring_size(size, align));
200 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,

--- 184 unchanged lines hidden (view full) ---

385{
386
387 return (vq->vq_free_cnt == 0);
388}
389
390void
391virtqueue_notify(struct virtqueue *vq)
392{
393 /* Ensure updated avail->idx is visible to host. */
394 mb();
387
395
396 if (vq_ring_must_notify_host(vq))
397 vq_ring_notify_host(vq);
388 vq->vq_queued_cnt = 0;
398 vq->vq_queued_cnt = 0;
389 vq_ring_notify_host(vq, 0);
390}
391
392int
393virtqueue_nused(struct virtqueue *vq)
394{
395 uint16_t used_idx, nused;
396
397 used_idx = vq->vq_ring.used->idx;
399}
400
401int
402virtqueue_nused(struct virtqueue *vq)
403{
404 uint16_t used_idx, nused;
405
406 used_idx = vq->vq_ring.used->idx;
398 if (used_idx >= vq->vq_used_cons_idx)
399 nused = used_idx - vq->vq_used_cons_idx;
400 else
401 nused = UINT16_MAX - vq->vq_used_cons_idx +
402 used_idx + 1;
407
408 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
403 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
404
405 return (nused);
406}
407
408int
409virtqueue_intr(struct virtqueue *vq)
410{

--- 11 unchanged lines hidden (view full) ---

422virtqueue_enable_intr(struct virtqueue *vq)
423{
424
425 /*
426 * Enable interrupts, making sure we get the latest
427 * index of what's already been consumed.
428 */
429 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
409 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
410
411 return (nused);
412}
413
414int
415virtqueue_intr(struct virtqueue *vq)
416{

--- 11 unchanged lines hidden (view full) ---

428virtqueue_enable_intr(struct virtqueue *vq)
429{
430
431 /*
432 * Enable interrupts, making sure we get the latest
433 * index of what's already been consumed.
434 */
435 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
436 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
437 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx;
438 else
439 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
430
431 mb();
432
433 /*
434 * Additional items may have been consumed in the time between
435 * since we last checked and enabled interrupts above. Let our
436 * caller know so it processes the new entries.
437 */
438 if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
439 return (1);
440
441 return (0);
442}
443
440
441 mb();
442
443 /*
444 * Additional items may have been consumed in the time between
445 * since we last checked and enabled interrupts above. Let our
446 * caller know so it processes the new entries.
447 */
448 if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
449 return (1);
450
451 return (0);
452}
453
454int
455virtqueue_postpone_intr(struct virtqueue *vq)
456{
457 uint16_t ndesc;
458
459 /*
460 * Postpone until at least half of the available descriptors
461 * have been consumed.
462 *
463 * XXX Adaptive factor? (Linux uses 3/4)
464 */
465 ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2;
466
467 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
468 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
469 else
470 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
471
472 mb();
473
474 /*
475 * Enough items may have already been consumed to meet our
476 * threshold since we last checked. Let our caller know so
477 * it processes the new entries.
478 */
479 if (virtqueue_nused(vq) > ndesc)
480 return (1);
481
482 return (0);
483}
484
444void
445virtqueue_disable_intr(struct virtqueue *vq)
446{
447
448 /*
449 * Note this is only considered a hint to the host.
450 */
485void
486virtqueue_disable_intr(struct virtqueue *vq)
487{
488
489 /*
490 * Note this is only considered a hint to the host.
491 */
451 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
492 if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0)
493 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
452}
453
454int
455virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
456 int readable, int writable)
457{
458 struct vq_desc_extra *dxp;
459 int needed;

--- 153 unchanged lines hidden (view full) ---

613 * descriptor.
614 */
615 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
616 vq->vq_ring.avail->ring[avail_idx] = desc_idx;
617
618 mb();
619 vq->vq_ring.avail->idx++;
620
494}
495
496int
497virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
498 int readable, int writable)
499{
500 struct vq_desc_extra *dxp;
501 int needed;

--- 153 unchanged lines hidden (view full) ---

655 * descriptor.
656 */
657 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
658 vq->vq_ring.avail->ring[avail_idx] = desc_idx;
659
660 mb();
661 vq->vq_ring.avail->idx++;
662
621 /* Keep pending count until virtqueue_notify() for debugging. */
663 /* Keep pending count until virtqueue_notify(). */
622 vq->vq_queued_cnt++;
623}
624
625static uint16_t
626vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
627 uint16_t head_idx, struct sglist *sg, int readable, int writable)
628{
629 struct sglist_seg *seg;

--- 74 unchanged lines hidden (view full) ---

704 if (vq->vq_free_cnt == 0)
705 VQ_RING_ASSERT_CHAIN_TERM(vq);
706 else
707 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
708
709 vq_ring_update_avail(vq, head_idx);
710}
711
664 vq->vq_queued_cnt++;
665}
666
667static uint16_t
668vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
669 uint16_t head_idx, struct sglist *sg, int readable, int writable)
670{
671 struct sglist_seg *seg;

--- 74 unchanged lines hidden (view full) ---

746 if (vq->vq_free_cnt == 0)
747 VQ_RING_ASSERT_CHAIN_TERM(vq);
748 else
749 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
750
751 vq_ring_update_avail(vq, head_idx);
752}
753
712static void
713vq_ring_notify_host(struct virtqueue *vq, int force)
754static int
755vq_ring_must_notify_host(struct virtqueue *vq)
714{
756{
757 uint16_t new_idx, prev_idx, event_idx;
715
758
716 mb();
759 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
760 new_idx = vq->vq_ring.avail->idx;
761 prev_idx = new_idx - vq->vq_queued_cnt;
762 event_idx = vring_avail_event(&vq->vq_ring);
717
763
718 if (force ||
719 (vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0)
720 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
764 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
765 }
766
767 return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
721}
722
723static void
768}
769
770static void
771vq_ring_notify_host(struct virtqueue *vq)
772{
773
774 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
775}
776
777static void
724vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
725{
726 struct vring_desc *dp;
727 struct vq_desc_extra *dxp;
728
729 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
730 dp = &vq->vq_ring.desc[desc_idx];
731 dxp = &vq->vq_descx[desc_idx];

--- 24 unchanged lines hidden --- 		778vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
779{
780 struct vring_desc *dp;
781 struct vq_desc_extra *dxp;
782
783 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
784 dp = &vq->vq_ring.desc[desc_idx];
785 dxp = &vq->vq_descx[desc_idx];

--- 24 unchanged lines hidden ---