virtqueue.c (234270) | virtqueue.c (238360) |
---|---|
1/*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 16 unchanged lines hidden (view full) --- 25 */ 26 27/* 28 * Implements the virtqueue interface as basically described 29 * in the original VirtIO paper. 30 */ 31 32#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 16 unchanged lines hidden (view full) --- 25 */ 26 27/* 28 * Implements the virtqueue interface as basically described 29 * in the original VirtIO paper. 30 */ 31 32#include <sys/cdefs.h> |
33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 234270 2012-04-14 05:48:04Z grehan $"); | 33__FBSDID("$FreeBSD: head/sys/dev/virtio/virtqueue.c 238360 2012-07-11 02:57:19Z grehan $"); |
34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/malloc.h> 39#include <sys/sglist.h> 40#include <vm/vm.h> 41#include <vm/pmap.h> --- 80 unchanged lines hidden (view full) --- 122 123static void vq_ring_init(struct virtqueue *); 124static void vq_ring_update_avail(struct virtqueue *, uint16_t); 125static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 126 struct vring_desc *, uint16_t, struct sglist *, int, int); 127static int vq_ring_use_indirect(struct virtqueue *, int); 128static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 129 struct sglist *, int, int); | 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/malloc.h> 39#include <sys/sglist.h> 40#include <vm/vm.h> 41#include <vm/pmap.h> --- 80 unchanged lines hidden (view full) --- 122 123static void vq_ring_init(struct virtqueue *); 124static void vq_ring_update_avail(struct virtqueue *, uint16_t); 125static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 126 struct vring_desc *, uint16_t, struct sglist *, int, int); 127static int vq_ring_use_indirect(struct virtqueue *, int); 128static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 129 struct sglist *, int, int); |
130static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); |
|
130static int vq_ring_must_notify_host(struct virtqueue *); 131static void vq_ring_notify_host(struct virtqueue *); 132static void vq_ring_free_chain(struct virtqueue *, uint16_t); 133 134uint64_t 135virtqueue_filter_features(uint64_t features) 136{ 137 uint64_t mask; --- 168 unchanged lines hidden (view full) --- 306 "%s: '%s' changed size; old=%hu, new=%hu\n", 307 __func__, vq->vq_name, vq->vq_nentries, size); 308 return (EINVAL); 309 } 310 311 /* Warn if the virtqueue was not properly cleaned up. */ 312 if (vq->vq_free_cnt != vq->vq_nentries) { 313 device_printf(vq->vq_dev, | 131static int vq_ring_must_notify_host(struct virtqueue *); 132static void vq_ring_notify_host(struct virtqueue *); 133static void vq_ring_free_chain(struct virtqueue *, uint16_t); 134 135uint64_t 136virtqueue_filter_features(uint64_t features) 137{ 138 uint64_t mask; --- 168 unchanged lines hidden (view full) --- 307 "%s: '%s' changed size; old=%hu, new=%hu\n", 308 __func__, vq->vq_name, vq->vq_nentries, size); 309 return (EINVAL); 310 } 311 312 /* Warn if the virtqueue was not properly cleaned up. */ 313 if (vq->vq_free_cnt != vq->vq_nentries) { 314 device_printf(vq->vq_dev, |
314 "%s: warning, '%s' virtqueue not empty, " | 315 "%s: warning '%s' virtqueue not empty, " |
315 "leaking %d entries\n", __func__, vq->vq_name, 316 vq->vq_nentries - vq->vq_free_cnt); 317 } 318 319 vq->vq_desc_head_idx = 0; 320 vq->vq_used_cons_idx = 0; 321 vq->vq_queued_cnt = 0; 322 vq->vq_free_cnt = vq->vq_nentries; --- 62 unchanged lines hidden (view full) --- 385{ 386 387 return (vq->vq_free_cnt == 0); 388} 389 390void 391virtqueue_notify(struct virtqueue *vq) 392{ | 316 "leaking %d entries\n", __func__, vq->vq_name, 317 vq->vq_nentries - vq->vq_free_cnt); 318 } 319 320 vq->vq_desc_head_idx = 0; 321 vq->vq_used_cons_idx = 0; 322 vq->vq_queued_cnt = 0; 323 vq->vq_free_cnt = vq->vq_nentries; --- 62 unchanged lines hidden (view full) --- 386{ 387 388 return (vq->vq_free_cnt == 0); 389} 390 391void 392virtqueue_notify(struct virtqueue *vq) 393{ |
394 |
|
393 /* Ensure updated avail->idx is visible to host. */ 394 mb(); 395 396 if (vq_ring_must_notify_host(vq)) 397 vq_ring_notify_host(vq); 398 vq->vq_queued_cnt = 0; 399} 400 --- 22 unchanged lines hidden (view full) --- 423 424 return (1); 425} 426 427int 428virtqueue_enable_intr(struct virtqueue *vq) 429{ 430 | 395 /* Ensure updated avail->idx is visible to host. */ 396 mb(); 397 398 if (vq_ring_must_notify_host(vq)) 399 vq_ring_notify_host(vq); 400 vq->vq_queued_cnt = 0; 401} 402 --- 22 unchanged lines hidden (view full) --- 425 426 return (1); 427} 428 429int 430virtqueue_enable_intr(struct virtqueue *vq) 431{ 432 |
431 /* 432 * Enable interrupts, making sure we get the latest 433 * index of what's already been consumed. 434 */ 435 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 436 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 437 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx; 438 else 439 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 440 441 mb(); 442 443 /* 444 * Additional items may have been consumed in the time between 445 * since we last checked and enabled interrupts above. Let our 446 * caller know so it processes the new entries. 447 */ 448 if (vq->vq_used_cons_idx != vq->vq_ring.used->idx) 449 return (1); 450 451 return (0); | 433 return (vq_ring_enable_interrupt(vq, 0)); |
452} 453 454int 455virtqueue_postpone_intr(struct virtqueue *vq) 456{ | 434} 435 436int 437virtqueue_postpone_intr(struct virtqueue *vq) 438{ |
457 uint16_t ndesc; | 439 uint16_t ndesc, avail_idx; |
458 459 /* | 440 441 /* |
460 * Postpone until at least half of the available descriptors 461 * have been consumed. 462 * 463 * XXX Adaptive factor? (Linux uses 3/4) | 442 * Request the next interrupt be postponed until at least half 443 * of the available descriptors have been consumed. |
464 */ | 444 */ |
465 ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2; | 445 avail_idx = vq->vq_ring.avail->idx; 446 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx) / 2; |
466 | 447 |
467 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 468 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 469 else 470 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 471 472 mb(); 473 474 /* 475 * Enough items may have already been consumed to meet our 476 * threshold since we last checked. Let our caller know so 477 * it processes the new entries. 478 */ 479 if (virtqueue_nused(vq) > ndesc) 480 return (1); 481 482 return (0); | 448 return (vq_ring_enable_interrupt(vq, ndesc)); |
483} 484 485void 486virtqueue_disable_intr(struct virtqueue *vq) 487{ 488 489 /* 490 * Note this is only considered a hint to the host. --- 256 unchanged lines hidden (view full) --- 747 VQ_RING_ASSERT_CHAIN_TERM(vq); 748 else 749 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 750 751 vq_ring_update_avail(vq, head_idx); 752} 753 754static int | 449} 450 451void 452virtqueue_disable_intr(struct virtqueue *vq) 453{ 454 455 /* 456 * Note this is only considered a hint to the host. --- 256 unchanged lines hidden (view full) --- 713 VQ_RING_ASSERT_CHAIN_TERM(vq); 714 else 715 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 716 717 vq_ring_update_avail(vq, head_idx); 718} 719 720static int |
721vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) 722{ 723 724 /* 725 * Enable interrupts, making sure we get the latest index of 726 * what's already been consumed. 727 */ 728 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 729 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 730 else 731 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 732 733 mb(); 734 735 /* 736 * Enough items may have already been consumed to meet our threshold 737 * since we last checked. Let our caller know so it processes the new 738 * entries. 739 */ 740 if (virtqueue_nused(vq) > ndesc) 741 return (1); 742 743 return (0); 744} 745 746static int |
|
755vq_ring_must_notify_host(struct virtqueue *vq) 756{ 757 uint16_t new_idx, prev_idx, event_idx; 758 759 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 760 new_idx = vq->vq_ring.avail->idx; 761 prev_idx = new_idx - vq->vq_queued_cnt; 762 event_idx = vring_avail_event(&vq->vq_ring); --- 20 unchanged lines hidden (view full) --- 783 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 784 dp = &vq->vq_ring.desc[desc_idx]; 785 dxp = &vq->vq_descx[desc_idx]; 786 787 if (vq->vq_free_cnt == 0) 788 VQ_RING_ASSERT_CHAIN_TERM(vq); 789 790 vq->vq_free_cnt += dxp->ndescs; | 747vq_ring_must_notify_host(struct virtqueue *vq) 748{ 749 uint16_t new_idx, prev_idx, event_idx; 750 751 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 752 new_idx = vq->vq_ring.avail->idx; 753 prev_idx = new_idx - vq->vq_queued_cnt; 754 event_idx = vring_avail_event(&vq->vq_ring); --- 20 unchanged lines hidden (view full) --- 775 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 776 dp = &vq->vq_ring.desc[desc_idx]; 777 dxp = &vq->vq_descx[desc_idx]; 778 779 if (vq->vq_free_cnt == 0) 780 VQ_RING_ASSERT_CHAIN_TERM(vq); 781 782 vq->vq_free_cnt += dxp->ndescs; |
791 dxp->ndescs--; | |
792 | 783 |
784#ifdef INVARIANTS |
|
793 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 794 while (dp->flags & VRING_DESC_F_NEXT) { 795 VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 796 dp = &vq->vq_ring.desc[dp->next]; 797 dxp->ndescs--; 798 } 799 } | 785 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 786 while (dp->flags & VRING_DESC_F_NEXT) { 787 VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 788 dp = &vq->vq_ring.desc[dp->next]; 789 dxp->ndescs--; 790 } 791 } |
800 VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain"); | 792 VQASSERT(vq, dxp->ndescs == 1, 793 "failed to free entire desc chain, remaining: %d", dxp->ndescs); 794#endif 795 dxp->ndescs = 0; |
801 802 /* 803 * We must append the existing free chain, if any, to the end of 804 * newly freed chain. If the virtqueue was completely used, then 805 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 806 */ 807 dp->next = vq->vq_desc_head_idx; 808 vq->vq_desc_head_idx = desc_idx; 809} | 796 797 /* 798 * We must append the existing free chain, if any, to the end of 799 * newly freed chain. If the virtqueue was completely used, then 800 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 801 */ 802 dp->next = vq->vq_desc_head_idx; 803 vq->vq_desc_head_idx = desc_idx; 804} |