Deleted Added
sdiff udiff text old ( 234270 ) new ( 238360 )
full compact
virtqueue.c (234270) virtqueue.c (238360)
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * Implements the virtqueue interface as basically described
29 * in the original VirtIO paper.
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 16 unchanged lines hidden (view full) ---

25 */
26
27/*
28 * Implements the virtqueue interface as basically described
29 * in the original VirtIO paper.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 234270 2012-04-14 05:48:04Z grehan $");
33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 238360 2012-07-11 02:57:19Z grehan $");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/sglist.h>
40#include <vm/vm.h>
41#include <vm/pmap.h>

--- 80 unchanged lines hidden (view full) ---

122
123static void vq_ring_init(struct virtqueue *);
124static void vq_ring_update_avail(struct virtqueue *, uint16_t);
125static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
126 struct vring_desc *, uint16_t, struct sglist *, int, int);
127static int vq_ring_use_indirect(struct virtqueue *, int);
128static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
129 struct sglist *, int, int);
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/sglist.h>
40#include <vm/vm.h>
41#include <vm/pmap.h>

--- 80 unchanged lines hidden (view full) ---

122
123static void vq_ring_init(struct virtqueue *);
124static void vq_ring_update_avail(struct virtqueue *, uint16_t);
125static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
126 struct vring_desc *, uint16_t, struct sglist *, int, int);
127static int vq_ring_use_indirect(struct virtqueue *, int);
128static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
129 struct sglist *, int, int);
130static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
130static int vq_ring_must_notify_host(struct virtqueue *);
131static void vq_ring_notify_host(struct virtqueue *);
132static void vq_ring_free_chain(struct virtqueue *, uint16_t);
133
134uint64_t
135virtqueue_filter_features(uint64_t features)
136{
137 uint64_t mask;

--- 168 unchanged lines hidden (view full) ---

306 "%s: '%s' changed size; old=%hu, new=%hu\n",
307 __func__, vq->vq_name, vq->vq_nentries, size);
308 return (EINVAL);
309 }
310
311 /* Warn if the virtqueue was not properly cleaned up. */
312 if (vq->vq_free_cnt != vq->vq_nentries) {
313 device_printf(vq->vq_dev,
131static int vq_ring_must_notify_host(struct virtqueue *);
132static void vq_ring_notify_host(struct virtqueue *);
133static void vq_ring_free_chain(struct virtqueue *, uint16_t);
134
135uint64_t
136virtqueue_filter_features(uint64_t features)
137{
138 uint64_t mask;

--- 168 unchanged lines hidden (view full) ---

307 "%s: '%s' changed size; old=%hu, new=%hu\n",
308 __func__, vq->vq_name, vq->vq_nentries, size);
309 return (EINVAL);
310 }
311
312 /* Warn if the virtqueue was not properly cleaned up. */
313 if (vq->vq_free_cnt != vq->vq_nentries) {
314 device_printf(vq->vq_dev,
314 "%s: warning, '%s' virtqueue not empty, "
315 "%s: warning '%s' virtqueue not empty, "
315 "leaking %d entries\n", __func__, vq->vq_name,
316 vq->vq_nentries - vq->vq_free_cnt);
317 }
318
319 vq->vq_desc_head_idx = 0;
320 vq->vq_used_cons_idx = 0;
321 vq->vq_queued_cnt = 0;
322 vq->vq_free_cnt = vq->vq_nentries;

--- 62 unchanged lines hidden (view full) ---

385{
386
387 return (vq->vq_free_cnt == 0);
388}
389
390void
391virtqueue_notify(struct virtqueue *vq)
392{
316 "leaking %d entries\n", __func__, vq->vq_name,
317 vq->vq_nentries - vq->vq_free_cnt);
318 }
319
320 vq->vq_desc_head_idx = 0;
321 vq->vq_used_cons_idx = 0;
322 vq->vq_queued_cnt = 0;
323 vq->vq_free_cnt = vq->vq_nentries;

--- 62 unchanged lines hidden (view full) ---

386{
387
388 return (vq->vq_free_cnt == 0);
389}
390
391void
392virtqueue_notify(struct virtqueue *vq)
393{
394
393 /* Ensure updated avail->idx is visible to host. */
394 mb();
395
396 if (vq_ring_must_notify_host(vq))
397 vq_ring_notify_host(vq);
398 vq->vq_queued_cnt = 0;
399}
400

--- 22 unchanged lines hidden (view full) ---

423
424 return (1);
425}
426
427int
428virtqueue_enable_intr(struct virtqueue *vq)
429{
430
395 /* Ensure updated avail->idx is visible to host. */
396 mb();
397
398 if (vq_ring_must_notify_host(vq))
399 vq_ring_notify_host(vq);
400 vq->vq_queued_cnt = 0;
401}
402

--- 22 unchanged lines hidden (view full) ---

425
426 return (1);
427}
428
429int
430virtqueue_enable_intr(struct virtqueue *vq)
431{
432
431 /*
432 * Enable interrupts, making sure we get the latest
433 * index of what's already been consumed.
434 */
435 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
436 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
437 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx;
438 else
439 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
440
441 mb();
442
443 /*
444 * Additional items may have been consumed in the time between
445 * since we last checked and enabled interrupts above. Let our
446 * caller know so it processes the new entries.
447 */
448 if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
449 return (1);
450
451 return (0);
433 return (vq_ring_enable_interrupt(vq, 0));
452}
453
454int
455virtqueue_postpone_intr(struct virtqueue *vq)
456{
434}
435
436int
437virtqueue_postpone_intr(struct virtqueue *vq)
438{
457 uint16_t ndesc;
439 uint16_t ndesc, avail_idx;
458
459 /*
440
441 /*
460 * Postpone until at least half of the available descriptors
461 * have been consumed.
462 *
463 * XXX Adaptive factor? (Linux uses 3/4)
442 * Request the next interrupt be postponed until at least half
443 * of the available descriptors have been consumed.
464 */
444 */
465 ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2;
445 avail_idx = vq->vq_ring.avail->idx;
446 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx) / 2;
466
447
467 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
468 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
469 else
470 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
471
472 mb();
473
474 /*
475 * Enough items may have already been consumed to meet our
476 * threshold since we last checked. Let our caller know so
477 * it processes the new entries.
478 */
479 if (virtqueue_nused(vq) > ndesc)
480 return (1);
481
482 return (0);
448 return (vq_ring_enable_interrupt(vq, ndesc));
483}
484
485void
486virtqueue_disable_intr(struct virtqueue *vq)
487{
488
489 /*
490 * Note this is only considered a hint to the host.

--- 256 unchanged lines hidden (view full) ---

747 VQ_RING_ASSERT_CHAIN_TERM(vq);
748 else
749 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
750
751 vq_ring_update_avail(vq, head_idx);
752}
753
754static int
449}
450
451void
452virtqueue_disable_intr(struct virtqueue *vq)
453{
454
455 /*
456 * Note this is only considered a hint to the host.

--- 256 unchanged lines hidden (view full) ---

713 VQ_RING_ASSERT_CHAIN_TERM(vq);
714 else
715 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
716
717 vq_ring_update_avail(vq, head_idx);
718}
719
720static int
721vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
722{
723
724 /*
725 * Enable interrupts, making sure we get the latest index of
726 * what's already been consumed.
727 */
728 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
729 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
730 else
731 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
732
733 mb();
734
735 /*
736 * Enough items may have already been consumed to meet our threshold
737 * since we last checked. Let our caller know so it processes the new
738 * entries.
739 */
740 if (virtqueue_nused(vq) > ndesc)
741 return (1);
742
743 return (0);
744}
745
746static int
755vq_ring_must_notify_host(struct virtqueue *vq)
756{
757 uint16_t new_idx, prev_idx, event_idx;
758
759 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
760 new_idx = vq->vq_ring.avail->idx;
761 prev_idx = new_idx - vq->vq_queued_cnt;
762 event_idx = vring_avail_event(&vq->vq_ring);

--- 20 unchanged lines hidden (view full) ---

783 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
784 dp = &vq->vq_ring.desc[desc_idx];
785 dxp = &vq->vq_descx[desc_idx];
786
787 if (vq->vq_free_cnt == 0)
788 VQ_RING_ASSERT_CHAIN_TERM(vq);
789
790 vq->vq_free_cnt += dxp->ndescs;
747vq_ring_must_notify_host(struct virtqueue *vq)
748{
749 uint16_t new_idx, prev_idx, event_idx;
750
751 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
752 new_idx = vq->vq_ring.avail->idx;
753 prev_idx = new_idx - vq->vq_queued_cnt;
754 event_idx = vring_avail_event(&vq->vq_ring);

--- 20 unchanged lines hidden (view full) ---

775 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
776 dp = &vq->vq_ring.desc[desc_idx];
777 dxp = &vq->vq_descx[desc_idx];
778
779 if (vq->vq_free_cnt == 0)
780 VQ_RING_ASSERT_CHAIN_TERM(vq);
781
782 vq->vq_free_cnt += dxp->ndescs;
791 dxp->ndescs--;
792
783
784#ifdef INVARIANTS
793 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
794 while (dp->flags & VRING_DESC_F_NEXT) {
795 VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
796 dp = &vq->vq_ring.desc[dp->next];
797 dxp->ndescs--;
798 }
799 }
785 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
786 while (dp->flags & VRING_DESC_F_NEXT) {
787 VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
788 dp = &vq->vq_ring.desc[dp->next];
789 dxp->ndescs--;
790 }
791 }
800 VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
792 VQASSERT(vq, dxp->ndescs == 1,
793 "failed to free entire desc chain, remaining: %d", dxp->ndescs);
794#endif
795 dxp->ndescs = 0;
801
802 /*
803 * We must append the existing free chain, if any, to the end of
804 * newly freed chain. If the virtqueue was completely used, then
805 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
806 */
807 dp->next = vq->vq_desc_head_idx;
808 vq->vq_desc_head_idx = desc_idx;
809}
796
797 /*
798 * We must append the existing free chain, if any, to the end of
799 * newly freed chain. If the virtqueue was completely used, then
800 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
801 */
802 dp->next = vq->vq_desc_head_idx;
803 vq->vq_desc_head_idx = desc_idx;
804}