1227652Sgrehan/*- 2252707Sbryanv * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3227652Sgrehan * All rights reserved. 4227652Sgrehan * 5227652Sgrehan * Redistribution and use in source and binary forms, with or without 6227652Sgrehan * modification, are permitted provided that the following conditions 7227652Sgrehan * are met: 8227652Sgrehan * 1. Redistributions of source code must retain the above copyright 9227652Sgrehan * notice unmodified, this list of conditions, and the following 10227652Sgrehan * disclaimer. 11227652Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 12227652Sgrehan * notice, this list of conditions and the following disclaimer in the 13227652Sgrehan * documentation and/or other materials provided with the distribution. 14227652Sgrehan * 15227652Sgrehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16227652Sgrehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17227652Sgrehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18227652Sgrehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19227652Sgrehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20227652Sgrehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21227652Sgrehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22227652Sgrehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23227652Sgrehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24227652Sgrehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25227652Sgrehan */ 26227652Sgrehan 27227652Sgrehan/* 28227652Sgrehan * Implements the virtqueue interface as basically described 29227652Sgrehan * in the original VirtIO paper. 30227652Sgrehan */ 31227652Sgrehan 32227652Sgrehan#include <sys/cdefs.h> 33227652Sgrehan__FBSDID("$FreeBSD$"); 34227652Sgrehan 35227652Sgrehan#include <sys/param.h> 36227652Sgrehan#include <sys/systm.h> 37227652Sgrehan#include <sys/kernel.h> 38227652Sgrehan#include <sys/malloc.h> 39227652Sgrehan#include <sys/sglist.h> 40227652Sgrehan#include <vm/vm.h> 41227652Sgrehan#include <vm/pmap.h> 42227652Sgrehan 43227652Sgrehan#include <machine/cpu.h> 44227652Sgrehan#include <machine/bus.h> 45227652Sgrehan#include <machine/atomic.h> 46227652Sgrehan#include <machine/resource.h> 47227652Sgrehan#include <sys/bus.h> 48227652Sgrehan#include <sys/rman.h> 49227652Sgrehan 50227652Sgrehan#include <dev/virtio/virtio.h> 51227652Sgrehan#include <dev/virtio/virtqueue.h> 52227652Sgrehan#include <dev/virtio/virtio_ring.h> 53227652Sgrehan 54227652Sgrehan#include "virtio_bus_if.h" 55227652Sgrehan 56227652Sgrehanstruct virtqueue { 57227652Sgrehan device_t vq_dev; 58227652Sgrehan char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 59227652Sgrehan uint16_t vq_queue_index; 60227652Sgrehan uint16_t vq_nentries; 61227652Sgrehan uint32_t vq_flags; 62227652Sgrehan#define VIRTQUEUE_FLAG_INDIRECT 0x0001 63234270Sgrehan#define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 64227652Sgrehan 65227652Sgrehan int vq_alignment; 66227652Sgrehan int vq_ring_size; 67227652Sgrehan void *vq_ring_mem; 68227652Sgrehan int vq_max_indirect_size; 69227652Sgrehan int vq_indirect_mem_size; 70227652Sgrehan virtqueue_intr_t *vq_intrhand; 71227652Sgrehan void *vq_intrhand_arg; 72227652Sgrehan 73227652Sgrehan struct vring vq_ring; 74227652Sgrehan uint16_t vq_free_cnt; 75227652Sgrehan uint16_t vq_queued_cnt; 76227652Sgrehan /* 77227652Sgrehan * Head of the free chain in the descriptor table. If 78227652Sgrehan * there are no free descriptors, this will be set to 79227652Sgrehan * VQ_RING_DESC_CHAIN_END. 80227652Sgrehan */ 81227652Sgrehan uint16_t vq_desc_head_idx; 82227652Sgrehan /* 83227652Sgrehan * Last consumed descriptor in the used table, 84227652Sgrehan * trails vq_ring.used->idx. 85227652Sgrehan */ 86227652Sgrehan uint16_t vq_used_cons_idx; 87227652Sgrehan 88227652Sgrehan struct vq_desc_extra { 89227652Sgrehan void *cookie; 90227652Sgrehan struct vring_desc *indirect; 91227652Sgrehan vm_paddr_t indirect_paddr; 92227652Sgrehan uint16_t ndescs; 93227652Sgrehan } vq_descx[0]; 94227652Sgrehan}; 95227652Sgrehan 96227652Sgrehan/* 97227652Sgrehan * The maximum virtqueue size is 2^15. Use that value as the end of 98227652Sgrehan * descriptor chain terminator since it will never be a valid index 99227652Sgrehan * in the descriptor table. This is used to verify we are correctly 100227652Sgrehan * handling vq_free_cnt. 101227652Sgrehan */ 102227652Sgrehan#define VQ_RING_DESC_CHAIN_END 32768 103227652Sgrehan 104227652Sgrehan#define VQASSERT(_vq, _exp, _msg, ...) \ 105227652Sgrehan KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ 106227652Sgrehan ##__VA_ARGS__)) 107227652Sgrehan 108227652Sgrehan#define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ 109227652Sgrehan VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ 110227652Sgrehan "invalid ring index: %d, max: %d", (_idx), \ 111227652Sgrehan (_vq)->vq_nentries) 112227652Sgrehan 113227652Sgrehan#define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ 114227652Sgrehan VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ 115227652Sgrehan VQ_RING_DESC_CHAIN_END, "full ring terminated " \ 116227652Sgrehan "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) 117227652Sgrehan 118227652Sgrehanstatic int virtqueue_init_indirect(struct virtqueue *vq, int); 119227652Sgrehanstatic void virtqueue_free_indirect(struct virtqueue *vq); 120227652Sgrehanstatic void virtqueue_init_indirect_list(struct virtqueue *, 121227652Sgrehan struct vring_desc *); 122227652Sgrehan 123227652Sgrehanstatic void vq_ring_init(struct virtqueue *); 124227652Sgrehanstatic void vq_ring_update_avail(struct virtqueue *, uint16_t); 125227652Sgrehanstatic uint16_t vq_ring_enqueue_segments(struct virtqueue *, 126227652Sgrehan struct vring_desc *, uint16_t, struct sglist *, int, int); 127227652Sgrehanstatic int vq_ring_use_indirect(struct virtqueue *, int); 128227652Sgrehanstatic void vq_ring_enqueue_indirect(struct virtqueue *, void *, 129227652Sgrehan struct sglist *, int, int); 130255109Sbryanvstatic int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); 131234270Sgrehanstatic int vq_ring_must_notify_host(struct virtqueue *); 132234270Sgrehanstatic void vq_ring_notify_host(struct virtqueue *); 133227652Sgrehanstatic void vq_ring_free_chain(struct virtqueue *, uint16_t); 134227652Sgrehan 135227652Sgrehanuint64_t 136227652Sgrehanvirtqueue_filter_features(uint64_t features) 137227652Sgrehan{ 138227652Sgrehan uint64_t mask; 139227652Sgrehan 140227652Sgrehan mask = (1 << VIRTIO_TRANSPORT_F_START) - 1; 141227652Sgrehan mask |= VIRTIO_RING_F_INDIRECT_DESC; 142234270Sgrehan mask |= VIRTIO_RING_F_EVENT_IDX; 143227652Sgrehan 144227652Sgrehan return (features & mask); 145227652Sgrehan} 146227652Sgrehan 147227652Sgrehanint 148227652Sgrehanvirtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, 149227652Sgrehan vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) 150227652Sgrehan{ 151227652Sgrehan struct virtqueue *vq; 152227652Sgrehan int error; 153227652Sgrehan 154227652Sgrehan *vqp = NULL; 155227652Sgrehan error = 0; 156227652Sgrehan 157227652Sgrehan if (size == 0) { 158227652Sgrehan device_printf(dev, 159227652Sgrehan "virtqueue %d (%s) does not exist (size is zero)\n", 160227652Sgrehan queue, info->vqai_name); 161227652Sgrehan return (ENODEV); 162227652Sgrehan } else if (!powerof2(size)) { 163227652Sgrehan device_printf(dev, 164227652Sgrehan "virtqueue %d (%s) size is not a power of 2: %d\n", 165227652Sgrehan queue, info->vqai_name, size); 166227652Sgrehan return (ENXIO); 167227652Sgrehan } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { 168227652Sgrehan device_printf(dev, "virtqueue %d (%s) requested too many " 169227652Sgrehan "indirect descriptors: %d, max %d\n", 170227652Sgrehan queue, info->vqai_name, info->vqai_maxindirsz, 171227652Sgrehan VIRTIO_MAX_INDIRECT); 172227652Sgrehan return (EINVAL); 173227652Sgrehan } 174227652Sgrehan 175227652Sgrehan vq = malloc(sizeof(struct virtqueue) + 176227652Sgrehan size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); 177227652Sgrehan if (vq == NULL) { 178227652Sgrehan device_printf(dev, "cannot allocate virtqueue\n"); 179227652Sgrehan return (ENOMEM); 180227652Sgrehan } 181227652Sgrehan 182227652Sgrehan vq->vq_dev = dev; 183227652Sgrehan strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); 184227652Sgrehan vq->vq_queue_index = queue; 185227652Sgrehan vq->vq_alignment = align; 186227652Sgrehan vq->vq_nentries = size; 187227652Sgrehan vq->vq_free_cnt = size; 188227652Sgrehan vq->vq_intrhand = info->vqai_intr; 189227652Sgrehan vq->vq_intrhand_arg = info->vqai_intr_arg; 190227652Sgrehan 191234270Sgrehan if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) 192234270Sgrehan vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; 193234270Sgrehan 194227652Sgrehan if (info->vqai_maxindirsz > 1) { 195227652Sgrehan error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); 196227652Sgrehan if (error) 197227652Sgrehan goto fail; 198227652Sgrehan } 199227652Sgrehan 200227652Sgrehan vq->vq_ring_size = round_page(vring_size(size, align)); 201227652Sgrehan vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, 202227652Sgrehan M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); 203227652Sgrehan if (vq->vq_ring_mem == NULL) { 204227652Sgrehan device_printf(dev, 205227652Sgrehan "cannot allocate memory for virtqueue ring\n"); 206227652Sgrehan error = ENOMEM; 207227652Sgrehan goto fail; 208227652Sgrehan } 209227652Sgrehan 210227652Sgrehan vq_ring_init(vq); 211227652Sgrehan virtqueue_disable_intr(vq); 212227652Sgrehan 213227652Sgrehan *vqp = vq; 214227652Sgrehan 215227652Sgrehanfail: 216227652Sgrehan if (error) 217227652Sgrehan virtqueue_free(vq); 218227652Sgrehan 219227652Sgrehan return (error); 220227652Sgrehan} 221227652Sgrehan 222227652Sgrehanstatic int 223227652Sgrehanvirtqueue_init_indirect(struct virtqueue *vq, int indirect_size) 224227652Sgrehan{ 225227652Sgrehan device_t dev; 226227652Sgrehan struct vq_desc_extra *dxp; 227227652Sgrehan int i, size; 228227652Sgrehan 229227652Sgrehan dev = vq->vq_dev; 230227652Sgrehan 231227652Sgrehan if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 232227652Sgrehan /* 233227652Sgrehan * Indirect descriptors requested by the driver but not 234227652Sgrehan * negotiated. Return zero to keep the initialization 235227652Sgrehan * going: we'll run fine without. 236227652Sgrehan */ 237227652Sgrehan if (bootverbose) 238227652Sgrehan device_printf(dev, "virtqueue %d (%s) requested " 239227652Sgrehan "indirect descriptors but not negotiated\n", 240227652Sgrehan vq->vq_queue_index, vq->vq_name); 241227652Sgrehan return (0); 242227652Sgrehan } 243227652Sgrehan 244227652Sgrehan size = indirect_size * sizeof(struct vring_desc); 245227652Sgrehan vq->vq_max_indirect_size = indirect_size; 246227652Sgrehan vq->vq_indirect_mem_size = size; 247227652Sgrehan vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; 248227652Sgrehan 249227652Sgrehan for (i = 0; i < vq->vq_nentries; i++) { 250227652Sgrehan dxp = &vq->vq_descx[i]; 251227652Sgrehan 252227652Sgrehan dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); 253227652Sgrehan if (dxp->indirect == NULL) { 254227652Sgrehan device_printf(dev, "cannot allocate indirect list\n"); 255227652Sgrehan return (ENOMEM); 256227652Sgrehan } 257227652Sgrehan 258227652Sgrehan dxp->indirect_paddr = vtophys(dxp->indirect); 259227652Sgrehan virtqueue_init_indirect_list(vq, dxp->indirect); 260227652Sgrehan } 261227652Sgrehan 262227652Sgrehan return (0); 263227652Sgrehan} 264227652Sgrehan 265227652Sgrehanstatic void 266227652Sgrehanvirtqueue_free_indirect(struct virtqueue *vq) 267227652Sgrehan{ 268227652Sgrehan struct vq_desc_extra *dxp; 269227652Sgrehan int i; 270227652Sgrehan 271227652Sgrehan for (i = 0; i < vq->vq_nentries; i++) { 272227652Sgrehan dxp = &vq->vq_descx[i]; 273227652Sgrehan 274227652Sgrehan if (dxp->indirect == NULL) 275227652Sgrehan break; 276227652Sgrehan 277227652Sgrehan free(dxp->indirect, M_DEVBUF); 278227652Sgrehan dxp->indirect = NULL; 279227652Sgrehan dxp->indirect_paddr = 0; 280227652Sgrehan } 281227652Sgrehan 282227652Sgrehan vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; 283227652Sgrehan vq->vq_indirect_mem_size = 0; 284227652Sgrehan} 285227652Sgrehan 286227652Sgrehanstatic void 287227652Sgrehanvirtqueue_init_indirect_list(struct virtqueue *vq, 288227652Sgrehan struct vring_desc *indirect) 289227652Sgrehan{ 290227652Sgrehan int i; 291227652Sgrehan 292227652Sgrehan bzero(indirect, vq->vq_indirect_mem_size); 293227652Sgrehan 294227652Sgrehan for (i = 0; i < vq->vq_max_indirect_size - 1; i++) 295227652Sgrehan indirect[i].next = i + 1; 296227652Sgrehan indirect[i].next = VQ_RING_DESC_CHAIN_END; 297227652Sgrehan} 298227652Sgrehan 299227652Sgrehanint 300227652Sgrehanvirtqueue_reinit(struct virtqueue *vq, uint16_t size) 301227652Sgrehan{ 302227652Sgrehan struct vq_desc_extra *dxp; 303227652Sgrehan int i; 304227652Sgrehan 305227652Sgrehan if (vq->vq_nentries != size) { 306227652Sgrehan device_printf(vq->vq_dev, 307227652Sgrehan "%s: '%s' changed size; old=%hu, new=%hu\n", 308227652Sgrehan __func__, vq->vq_name, vq->vq_nentries, size); 309227652Sgrehan return (EINVAL); 310227652Sgrehan } 311227652Sgrehan 312227652Sgrehan /* Warn if the virtqueue was not properly cleaned up. */ 313227652Sgrehan if (vq->vq_free_cnt != vq->vq_nentries) { 314227652Sgrehan device_printf(vq->vq_dev, 315238360Sgrehan "%s: warning '%s' virtqueue not empty, " 316227652Sgrehan "leaking %d entries\n", __func__, vq->vq_name, 317227652Sgrehan vq->vq_nentries - vq->vq_free_cnt); 318227652Sgrehan } 319227652Sgrehan 320227652Sgrehan vq->vq_desc_head_idx = 0; 321227652Sgrehan vq->vq_used_cons_idx = 0; 322227652Sgrehan vq->vq_queued_cnt = 0; 323227652Sgrehan vq->vq_free_cnt = vq->vq_nentries; 324227652Sgrehan 325227652Sgrehan /* To be safe, reset all our allocated memory. */ 326227652Sgrehan bzero(vq->vq_ring_mem, vq->vq_ring_size); 327227652Sgrehan for (i = 0; i < vq->vq_nentries; i++) { 328227652Sgrehan dxp = &vq->vq_descx[i]; 329227652Sgrehan dxp->cookie = NULL; 330227652Sgrehan dxp->ndescs = 0; 331227652Sgrehan if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 332227652Sgrehan virtqueue_init_indirect_list(vq, dxp->indirect); 333227652Sgrehan } 334227652Sgrehan 335227652Sgrehan vq_ring_init(vq); 336227652Sgrehan virtqueue_disable_intr(vq); 337227652Sgrehan 338227652Sgrehan return (0); 339227652Sgrehan} 340227652Sgrehan 341227652Sgrehanvoid 342227652Sgrehanvirtqueue_free(struct virtqueue *vq) 343227652Sgrehan{ 344227652Sgrehan 345227652Sgrehan if (vq->vq_free_cnt != vq->vq_nentries) { 346227652Sgrehan device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " 347227652Sgrehan "leaking %d entries\n", vq->vq_name, 348227652Sgrehan vq->vq_nentries - vq->vq_free_cnt); 349227652Sgrehan } 350227652Sgrehan 351227652Sgrehan if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 352227652Sgrehan virtqueue_free_indirect(vq); 353227652Sgrehan 354227652Sgrehan if (vq->vq_ring_mem != NULL) { 355227652Sgrehan contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF); 356227652Sgrehan vq->vq_ring_size = 0; 357227652Sgrehan vq->vq_ring_mem = NULL; 358227652Sgrehan } 359227652Sgrehan 360227652Sgrehan free(vq, M_DEVBUF); 361227652Sgrehan} 362227652Sgrehan 363227652Sgrehanvm_paddr_t 364227652Sgrehanvirtqueue_paddr(struct virtqueue *vq) 365227652Sgrehan{ 366227652Sgrehan 367227652Sgrehan return (vtophys(vq->vq_ring_mem)); 368227652Sgrehan} 369227652Sgrehan 370227652Sgrehanint 371227652Sgrehanvirtqueue_size(struct virtqueue *vq) 372227652Sgrehan{ 373227652Sgrehan 374227652Sgrehan return (vq->vq_nentries); 375227652Sgrehan} 376227652Sgrehan 377227652Sgrehanint 378227652Sgrehanvirtqueue_empty(struct virtqueue *vq) 379227652Sgrehan{ 380227652Sgrehan 381227652Sgrehan return (vq->vq_nentries == vq->vq_free_cnt); 382227652Sgrehan} 383227652Sgrehan 384227652Sgrehanint 385227652Sgrehanvirtqueue_full(struct virtqueue *vq) 386227652Sgrehan{ 387227652Sgrehan 388227652Sgrehan return (vq->vq_free_cnt == 0); 389227652Sgrehan} 390227652Sgrehan 391227652Sgrehanvoid 392227652Sgrehanvirtqueue_notify(struct virtqueue *vq) 393227652Sgrehan{ 394238360Sgrehan 395234270Sgrehan /* Ensure updated avail->idx is visible to host. */ 396234270Sgrehan mb(); 397227652Sgrehan 398234270Sgrehan if (vq_ring_must_notify_host(vq)) 399234270Sgrehan vq_ring_notify_host(vq); 400227652Sgrehan vq->vq_queued_cnt = 0; 401227652Sgrehan} 402227652Sgrehan 403227652Sgrehanint 404227652Sgrehanvirtqueue_nused(struct virtqueue *vq) 405227652Sgrehan{ 406227652Sgrehan uint16_t used_idx, nused; 407227652Sgrehan 408227652Sgrehan used_idx = vq->vq_ring.used->idx; 409234270Sgrehan 410234270Sgrehan nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); 411227652Sgrehan VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); 412227652Sgrehan 413227652Sgrehan return (nused); 414227652Sgrehan} 415227652Sgrehan 416227652Sgrehanint 417252702Sbryanvvirtqueue_intr_filter(struct virtqueue *vq) 418227652Sgrehan{ 419227652Sgrehan 420252702Sbryanv if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 421252702Sbryanv return (0); 422227652Sgrehan 423252702Sbryanv virtqueue_disable_intr(vq); 424227652Sgrehan 425227652Sgrehan return (1); 426227652Sgrehan} 427227652Sgrehan 428252702Sbryanvvoid 429252702Sbryanvvirtqueue_intr(struct virtqueue *vq) 430252702Sbryanv{ 431252702Sbryanv 432252708Sbryanv vq->vq_intrhand(vq->vq_intrhand_arg); 433252702Sbryanv} 434252702Sbryanv 435227652Sgrehanint 436227652Sgrehanvirtqueue_enable_intr(struct virtqueue *vq) 437227652Sgrehan{ 438227652Sgrehan 439238360Sgrehan return (vq_ring_enable_interrupt(vq, 0)); 440227652Sgrehan} 441227652Sgrehan 442234270Sgrehanint 443255109Sbryanvvirtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint) 444234270Sgrehan{ 445238360Sgrehan uint16_t ndesc, avail_idx; 446234270Sgrehan 447238360Sgrehan avail_idx = vq->vq_ring.avail->idx; 448255109Sbryanv ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx); 449234270Sgrehan 450255109Sbryanv switch (hint) { 451255109Sbryanv case VQ_POSTPONE_SHORT: 452255166Sbryanv ndesc = ndesc / 4; 453255109Sbryanv break; 454255109Sbryanv case VQ_POSTPONE_LONG: 455255166Sbryanv ndesc = (ndesc * 3) / 4; 456255109Sbryanv break; 457255109Sbryanv case VQ_POSTPONE_EMPTIED: 458255109Sbryanv break; 459255109Sbryanv } 460255109Sbryanv 461238360Sgrehan return (vq_ring_enable_interrupt(vq, ndesc)); 462234270Sgrehan} 463234270Sgrehan 464255109Sbryanv/* 465255109Sbryanv * Note this is only considered a hint to the host. 466255109Sbryanv */ 467227652Sgrehanvoid 468227652Sgrehanvirtqueue_disable_intr(struct virtqueue *vq) 469227652Sgrehan{ 470227652Sgrehan 471255109Sbryanv if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 472255109Sbryanv vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - 473255109Sbryanv vq->vq_nentries - 1; 474255109Sbryanv } else 475234270Sgrehan vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 476227652Sgrehan} 477227652Sgrehan 478227652Sgrehanint 479227652Sgrehanvirtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, 480227652Sgrehan int readable, int writable) 481227652Sgrehan{ 482227652Sgrehan struct vq_desc_extra *dxp; 483227652Sgrehan int needed; 484227652Sgrehan uint16_t head_idx, idx; 485227652Sgrehan 486227652Sgrehan needed = readable + writable; 487227652Sgrehan 488227652Sgrehan VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); 489227652Sgrehan VQASSERT(vq, needed == sg->sg_nseg, 490227652Sgrehan "segment count mismatch, %d, %d", needed, sg->sg_nseg); 491227652Sgrehan VQASSERT(vq, 492227652Sgrehan needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, 493227652Sgrehan "too many segments to enqueue: %d, %d/%d", needed, 494227652Sgrehan vq->vq_nentries, vq->vq_max_indirect_size); 495227652Sgrehan 496227652Sgrehan if (needed < 1) 497227652Sgrehan return (EINVAL); 498227652Sgrehan if (vq->vq_free_cnt == 0) 499227652Sgrehan return (ENOSPC); 500227652Sgrehan 501227652Sgrehan if (vq_ring_use_indirect(vq, needed)) { 502227652Sgrehan vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); 503227652Sgrehan return (0); 504227652Sgrehan } else if (vq->vq_free_cnt < needed) 505227652Sgrehan return (EMSGSIZE); 506227652Sgrehan 507227652Sgrehan head_idx = vq->vq_desc_head_idx; 508227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 509227652Sgrehan dxp = &vq->vq_descx[head_idx]; 510227652Sgrehan 511227652Sgrehan VQASSERT(vq, dxp->cookie == NULL, 512227652Sgrehan "cookie already exists for index %d", head_idx); 513227652Sgrehan dxp->cookie = cookie; 514227652Sgrehan dxp->ndescs = needed; 515227652Sgrehan 516227652Sgrehan idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, 517227652Sgrehan sg, readable, writable); 518227652Sgrehan 519227652Sgrehan vq->vq_desc_head_idx = idx; 520227652Sgrehan vq->vq_free_cnt -= needed; 521227652Sgrehan if (vq->vq_free_cnt == 0) 522227652Sgrehan VQ_RING_ASSERT_CHAIN_TERM(vq); 523227652Sgrehan else 524227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, idx); 525227652Sgrehan 526227652Sgrehan vq_ring_update_avail(vq, head_idx); 527227652Sgrehan 528227652Sgrehan return (0); 529227652Sgrehan} 530227652Sgrehan 531227652Sgrehanvoid * 532227652Sgrehanvirtqueue_dequeue(struct virtqueue *vq, uint32_t *len) 533227652Sgrehan{ 534227652Sgrehan struct vring_used_elem *uep; 535227652Sgrehan void *cookie; 536227652Sgrehan uint16_t used_idx, desc_idx; 537227652Sgrehan 538227652Sgrehan if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 539227652Sgrehan return (NULL); 540227652Sgrehan 541227652Sgrehan used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); 542227652Sgrehan uep = &vq->vq_ring.used->ring[used_idx]; 543227652Sgrehan 544240427Sgrehan rmb(); 545227652Sgrehan desc_idx = (uint16_t) uep->id; 546227652Sgrehan if (len != NULL) 547227652Sgrehan *len = uep->len; 548227652Sgrehan 549227652Sgrehan vq_ring_free_chain(vq, desc_idx); 550227652Sgrehan 551227652Sgrehan cookie = vq->vq_descx[desc_idx].cookie; 552227652Sgrehan VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); 553227652Sgrehan vq->vq_descx[desc_idx].cookie = NULL; 554227652Sgrehan 555227652Sgrehan return (cookie); 556227652Sgrehan} 557227652Sgrehan 558227652Sgrehanvoid * 559227652Sgrehanvirtqueue_poll(struct virtqueue *vq, uint32_t *len) 560227652Sgrehan{ 561227652Sgrehan void *cookie; 562227652Sgrehan 563227652Sgrehan while ((cookie = virtqueue_dequeue(vq, len)) == NULL) 564227652Sgrehan cpu_spinwait(); 565227652Sgrehan 566227652Sgrehan return (cookie); 567227652Sgrehan} 568227652Sgrehan 569227652Sgrehanvoid * 570227652Sgrehanvirtqueue_drain(struct virtqueue *vq, int *last) 571227652Sgrehan{ 572227652Sgrehan void *cookie; 573227652Sgrehan int idx; 574227652Sgrehan 575227652Sgrehan cookie = NULL; 576227652Sgrehan idx = *last; 577227652Sgrehan 578227652Sgrehan while (idx < vq->vq_nentries && cookie == NULL) { 579227652Sgrehan if ((cookie = vq->vq_descx[idx].cookie) != NULL) { 580227652Sgrehan vq->vq_descx[idx].cookie = NULL; 581227652Sgrehan /* Free chain to keep free count consistent. */ 582227652Sgrehan vq_ring_free_chain(vq, idx); 583227652Sgrehan } 584227652Sgrehan idx++; 585227652Sgrehan } 586227652Sgrehan 587227652Sgrehan *last = idx; 588227652Sgrehan 589227652Sgrehan return (cookie); 590227652Sgrehan} 591227652Sgrehan 592227652Sgrehanvoid 593227652Sgrehanvirtqueue_dump(struct virtqueue *vq) 594227652Sgrehan{ 595227652Sgrehan 596227652Sgrehan if (vq == NULL) 597227652Sgrehan return; 598227652Sgrehan 599227652Sgrehan printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " 600227652Sgrehan "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " 601227652Sgrehan "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n", 602227652Sgrehan vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, 603227652Sgrehan virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, 604227652Sgrehan vq->vq_ring.avail->idx, vq->vq_used_cons_idx, 605227652Sgrehan vq->vq_ring.used->idx, vq->vq_ring.avail->flags, 606227652Sgrehan vq->vq_ring.used->flags); 607227652Sgrehan} 608227652Sgrehan 609227652Sgrehanstatic void 610227652Sgrehanvq_ring_init(struct virtqueue *vq) 611227652Sgrehan{ 612227652Sgrehan struct vring *vr; 613227652Sgrehan char *ring_mem; 614227652Sgrehan int i, size; 615227652Sgrehan 616227652Sgrehan ring_mem = vq->vq_ring_mem; 617227652Sgrehan size = vq->vq_nentries; 618227652Sgrehan vr = &vq->vq_ring; 619227652Sgrehan 620227652Sgrehan vring_init(vr, size, ring_mem, vq->vq_alignment); 621227652Sgrehan 622227652Sgrehan for (i = 0; i < size - 1; i++) 623227652Sgrehan vr->desc[i].next = i + 1; 624227652Sgrehan vr->desc[i].next = VQ_RING_DESC_CHAIN_END; 625227652Sgrehan} 626227652Sgrehan 627227652Sgrehanstatic void 628227652Sgrehanvq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) 629227652Sgrehan{ 630227652Sgrehan uint16_t avail_idx; 631227652Sgrehan 632227652Sgrehan /* 633227652Sgrehan * Place the head of the descriptor chain into the next slot and make 634227652Sgrehan * it usable to the host. The chain is made available now rather than 635227652Sgrehan * deferring to virtqueue_notify() in the hopes that if the host is 636227652Sgrehan * currently running on another CPU, we can keep it processing the new 637227652Sgrehan * descriptor. 638227652Sgrehan */ 639227652Sgrehan avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); 640227652Sgrehan vq->vq_ring.avail->ring[avail_idx] = desc_idx; 641227652Sgrehan 642240427Sgrehan wmb(); 643227652Sgrehan vq->vq_ring.avail->idx++; 644227652Sgrehan 645234270Sgrehan /* Keep pending count until virtqueue_notify(). */ 646227652Sgrehan vq->vq_queued_cnt++; 647227652Sgrehan} 648227652Sgrehan 649227652Sgrehanstatic uint16_t 650227652Sgrehanvq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, 651227652Sgrehan uint16_t head_idx, struct sglist *sg, int readable, int writable) 652227652Sgrehan{ 653227652Sgrehan struct sglist_seg *seg; 654227652Sgrehan struct vring_desc *dp; 655227652Sgrehan int i, needed; 656227652Sgrehan uint16_t idx; 657227652Sgrehan 658227652Sgrehan needed = readable + writable; 659227652Sgrehan 660227652Sgrehan for (i = 0, idx = head_idx, seg = sg->sg_segs; 661227652Sgrehan i < needed; 662227652Sgrehan i++, idx = dp->next, seg++) { 663227652Sgrehan VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, 664227652Sgrehan "premature end of free desc chain"); 665227652Sgrehan 666227652Sgrehan dp = &desc[idx]; 667227652Sgrehan dp->addr = seg->ss_paddr; 668227652Sgrehan dp->len = seg->ss_len; 669227652Sgrehan dp->flags = 0; 670227652Sgrehan 671227652Sgrehan if (i < needed - 1) 672227652Sgrehan dp->flags |= VRING_DESC_F_NEXT; 673227652Sgrehan if (i >= readable) 674227652Sgrehan dp->flags |= VRING_DESC_F_WRITE; 675227652Sgrehan } 676227652Sgrehan 677227652Sgrehan return (idx); 678227652Sgrehan} 679227652Sgrehan 680227652Sgrehanstatic int 681227652Sgrehanvq_ring_use_indirect(struct virtqueue *vq, int needed) 682227652Sgrehan{ 683227652Sgrehan 684227652Sgrehan if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) 685227652Sgrehan return (0); 686227652Sgrehan 687227652Sgrehan if (vq->vq_max_indirect_size < needed) 688227652Sgrehan return (0); 689227652Sgrehan 690227652Sgrehan if (needed < 2) 691227652Sgrehan return (0); 692227652Sgrehan 693227652Sgrehan return (1); 694227652Sgrehan} 695227652Sgrehan 696227652Sgrehanstatic void 697227652Sgrehanvq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, 698227652Sgrehan struct sglist *sg, int readable, int writable) 699227652Sgrehan{ 700227652Sgrehan struct vring_desc *dp; 701227652Sgrehan struct vq_desc_extra *dxp; 702227652Sgrehan int needed; 703227652Sgrehan uint16_t head_idx; 704227652Sgrehan 705227652Sgrehan needed = readable + writable; 706227652Sgrehan VQASSERT(vq, needed <= vq->vq_max_indirect_size, 707227652Sgrehan "enqueuing too many indirect descriptors"); 708227652Sgrehan 709227652Sgrehan head_idx = vq->vq_desc_head_idx; 710227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 711227652Sgrehan dp = &vq->vq_ring.desc[head_idx]; 712227652Sgrehan dxp = &vq->vq_descx[head_idx]; 713227652Sgrehan 714227652Sgrehan VQASSERT(vq, dxp->cookie == NULL, 715227652Sgrehan "cookie already exists for index %d", head_idx); 716227652Sgrehan dxp->cookie = cookie; 717227652Sgrehan dxp->ndescs = 1; 718227652Sgrehan 719227652Sgrehan dp->addr = dxp->indirect_paddr; 720227652Sgrehan dp->len = needed * sizeof(struct vring_desc); 721227652Sgrehan dp->flags = VRING_DESC_F_INDIRECT; 722227652Sgrehan 723227652Sgrehan vq_ring_enqueue_segments(vq, dxp->indirect, 0, 724227652Sgrehan sg, readable, writable); 725227652Sgrehan 726227652Sgrehan vq->vq_desc_head_idx = dp->next; 727227652Sgrehan vq->vq_free_cnt--; 728227652Sgrehan if (vq->vq_free_cnt == 0) 729227652Sgrehan VQ_RING_ASSERT_CHAIN_TERM(vq); 730227652Sgrehan else 731227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 732227652Sgrehan 733227652Sgrehan vq_ring_update_avail(vq, head_idx); 734227652Sgrehan} 735227652Sgrehan 736234270Sgrehanstatic int 737238360Sgrehanvq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) 738238360Sgrehan{ 739238360Sgrehan 740238360Sgrehan /* 741238360Sgrehan * Enable interrupts, making sure we get the latest index of 742238360Sgrehan * what's already been consumed. 743238360Sgrehan */ 744238360Sgrehan if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 745238360Sgrehan vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 746238360Sgrehan else 747238360Sgrehan vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 748238360Sgrehan 749238360Sgrehan mb(); 750238360Sgrehan 751238360Sgrehan /* 752238360Sgrehan * Enough items may have already been consumed to meet our threshold 753238360Sgrehan * since we last checked. Let our caller know so it processes the new 754238360Sgrehan * entries. 755238360Sgrehan */ 756238360Sgrehan if (virtqueue_nused(vq) > ndesc) 757238360Sgrehan return (1); 758238360Sgrehan 759238360Sgrehan return (0); 760238360Sgrehan} 761238360Sgrehan 762238360Sgrehanstatic int 763234270Sgrehanvq_ring_must_notify_host(struct virtqueue *vq) 764227652Sgrehan{ 765234270Sgrehan uint16_t new_idx, prev_idx, event_idx; 766227652Sgrehan 767234270Sgrehan if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 768234270Sgrehan new_idx = vq->vq_ring.avail->idx; 769234270Sgrehan prev_idx = new_idx - vq->vq_queued_cnt; 770234270Sgrehan event_idx = vring_avail_event(&vq->vq_ring); 771227652Sgrehan 772234270Sgrehan return (vring_need_event(event_idx, new_idx, prev_idx) != 0); 773234270Sgrehan } 774234270Sgrehan 775234270Sgrehan return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); 776227652Sgrehan} 777227652Sgrehan 778227652Sgrehanstatic void 779234270Sgrehanvq_ring_notify_host(struct virtqueue *vq) 780234270Sgrehan{ 781234270Sgrehan 782234270Sgrehan VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); 783234270Sgrehan} 784234270Sgrehan 785234270Sgrehanstatic void 786227652Sgrehanvq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 787227652Sgrehan{ 788227652Sgrehan struct vring_desc *dp; 789227652Sgrehan struct vq_desc_extra *dxp; 790227652Sgrehan 791227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 792227652Sgrehan dp = &vq->vq_ring.desc[desc_idx]; 793227652Sgrehan dxp = &vq->vq_descx[desc_idx]; 794227652Sgrehan 795227652Sgrehan if (vq->vq_free_cnt == 0) 796227652Sgrehan VQ_RING_ASSERT_CHAIN_TERM(vq); 797227652Sgrehan 798227652Sgrehan vq->vq_free_cnt += dxp->ndescs; 799241469Sgrehan dxp->ndescs--; 800227652Sgrehan 801227652Sgrehan if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 802227652Sgrehan while (dp->flags & VRING_DESC_F_NEXT) { 803227652Sgrehan VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 804227652Sgrehan dp = &vq->vq_ring.desc[dp->next]; 805227652Sgrehan dxp->ndescs--; 806227652Sgrehan } 807227652Sgrehan } 808241469Sgrehan 809241469Sgrehan VQASSERT(vq, dxp->ndescs == 0, 810238360Sgrehan "failed to free entire desc chain, remaining: %d", dxp->ndescs); 811227652Sgrehan 812227652Sgrehan /* 813227652Sgrehan * We must append the existing free chain, if any, to the end of 814227652Sgrehan * newly freed chain. If the virtqueue was completely used, then 815227652Sgrehan * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 816227652Sgrehan */ 817227652Sgrehan dp->next = vq->vq_desc_head_idx; 818227652Sgrehan vq->vq_desc_head_idx = desc_idx; 819227652Sgrehan} 820