1/* $OpenBSD: virtio.c,v 1.25 2024/05/24 10:05:55 jsg Exp $ */ 2/* $NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $ */ 3 4/* 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/device.h> 33#include <sys/atomic.h> 34#include <sys/malloc.h> 35 36#include <dev/pv/virtioreg.h> 37#include <dev/pv/virtiovar.h> 38 39#if VIRTIO_DEBUG 40#define VIRTIO_ASSERT(x) KASSERT(x) 41#else 42#define VIRTIO_ASSERT(x) 43#endif 44 45void virtio_init_vq(struct virtio_softc *, 46 struct virtqueue *); 47void vq_free_entry(struct virtqueue *, struct vq_entry *); 48struct vq_entry *vq_alloc_entry(struct virtqueue *); 49 50struct cfdriver virtio_cd = { 51 NULL, "virtio", DV_DULL 52}; 53 54static const char * const virtio_device_name[] = { 55 "Unknown (0)", /* 0 */ 56 "Network", /* 1 */ 57 "Block", /* 2 */ 58 "Console", /* 3 */ 59 "Entropy", /* 4 */ 60 "Memory Balloon", /* 5 */ 61 "IO Memory", /* 6 */ 62 "Rpmsg", /* 7 */ 63 "SCSI host", /* 8 */ 64 "9P Transport", /* 9 */ 65 "mac80211 wlan", /* 10 */ 66 NULL, /* 11 */ 67 NULL, /* 12 */ 68 NULL, /* 13 */ 69 NULL, /* 14 */ 70 NULL, /* 15 */ 71 "GPU", /* 16 */ 72}; 73#define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 74 75const char * 76virtio_device_string(int id) 77{ 78 return id < NDEVNAMES ? virtio_device_name[id] : "Unknown"; 79} 80 81#if VIRTIO_DEBUG 82static const struct virtio_feature_name transport_feature_names[] = { 83 { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty"}, 84 { VIRTIO_F_RING_INDIRECT_DESC, "RingIndirectDesc"}, 85 { VIRTIO_F_RING_EVENT_IDX, "RingEventIdx"}, 86 { VIRTIO_F_BAD_FEATURE, "BadFeature"}, 87 { VIRTIO_F_VERSION_1, "Version1"}, 88 { 0, NULL} 89}; 90 91void 92virtio_log_features(uint64_t host, uint64_t neg, 93 const struct virtio_feature_name *guest_feature_names) 94{ 95 const struct virtio_feature_name *namep; 96 int i; 97 char c; 98 uint64_t bit; 99 100 for (i = 0; i < 64; i++) { 101 if (i == 30) { 102 /* 103 * VIRTIO_F_BAD_FEATURE is only used for 104 * checking correct negotiation 105 */ 106 continue; 107 } 108 bit = 1ULL << i; 109 if ((host&bit) == 0) 110 continue; 111 namep = guest_feature_names; 112 while (namep->bit && namep->bit != bit) 113 namep++; 114 if (namep->name == NULL) { 115 namep = transport_feature_names; 116 while (namep->bit && namep->bit != bit) 117 namep++; 118 } 119 c = (neg&bit) ? '+' : '-'; 120 if (namep->name) 121 printf(" %c%s", c, namep->name); 122 else 123 printf(" %cUnknown(%d)", c, i); 124 } 125} 126#endif 127 128/* 129 * Reset the device. 130 */ 131/* 132 * To reset the device to a known state, do following: 133 * virtio_reset(sc); // this will stop the device activity 134 * <dequeue finished requests>; // virtio_dequeue() still can be called 135 * <revoke pending requests in the vqs if any>; 136 * virtio_reinit_start(sc); // dequeue prohibited 137 * <some other initialization>; 138 * virtio_reinit_end(sc); // device activated; enqueue allowed 139 * Once attached, features are assumed to not change again. 140 */ 141void 142virtio_reset(struct virtio_softc *sc) 143{ 144 virtio_device_reset(sc); 145 sc->sc_active_features = 0; 146} 147 148void 149virtio_reinit_start(struct virtio_softc *sc) 150{ 151 int i; 152 153 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 154 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 155 virtio_negotiate_features(sc, NULL); 156 for (i = 0; i < sc->sc_nvqs; i++) { 157 int n; 158 struct virtqueue *vq = &sc->sc_vqs[i]; 159 n = virtio_read_queue_size(sc, vq->vq_index); 160 if (n == 0) /* vq disappeared */ 161 continue; 162 if (n != vq->vq_num) { 163 panic("%s: virtqueue size changed, vq index %d", 164 sc->sc_dev.dv_xname, vq->vq_index); 165 } 166 virtio_init_vq(sc, vq); 167 virtio_setup_queue(sc, vq, vq->vq_dmamap->dm_segs[0].ds_addr); 168 } 169} 170 171void 172virtio_reinit_end(struct virtio_softc *sc) 173{ 174 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 175} 176 177/* 178 * dmamap sync operations for a virtqueue. 179 */ 180static inline void 181vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 182{ 183 /* availoffset == sizeof(vring_desc)*vq_num */ 184 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 185 ops); 186} 187 188static inline void 189vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 190{ 191 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, vq->vq_availoffset, 192 offsetof(struct vring_avail, ring) + vq->vq_num * sizeof(uint16_t), 193 ops); 194} 195 196static inline void 197vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 198{ 199 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, vq->vq_usedoffset, 200 offsetof(struct vring_used, ring) + vq->vq_num * 201 sizeof(struct vring_used_elem), ops); 202} 203 204static inline void 205vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 206 int ops) 207{ 208 int offset = vq->vq_indirectoffset + 209 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 210 211 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, offset, 212 sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 213} 214 215/* 216 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 217 * and calls (*vq_done)() if some entries are consumed. 218 * For use in transport specific irq handlers. 219 */ 220int 221virtio_check_vqs(struct virtio_softc *sc) 222{ 223 int i, r = 0; 224 225 /* going backwards is better for if_vio */ 226 for (i = sc->sc_nvqs - 1; i >= 0; i--) 227 r |= virtio_check_vq(sc, &sc->sc_vqs[i]); 228 229 return r; 230} 231 232int 233virtio_check_vq(struct virtio_softc *sc, struct virtqueue *vq) 234{ 235 if (vq->vq_queued) { 236 vq->vq_queued = 0; 237 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 238 } 239 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 240 if (vq->vq_used_idx != vq->vq_used->idx) { 241 if (vq->vq_done) 242 return (vq->vq_done)(vq); 243 } 244 245 return 0; 246} 247 248/* 249 * Initialize vq structure. 250 */ 251void 252virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq) 253{ 254 int i, j; 255 int vq_size = vq->vq_num; 256 257 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 258 259 /* build the indirect descriptor chain */ 260 if (vq->vq_indirect != NULL) { 261 struct vring_desc *vd; 262 263 for (i = 0; i < vq_size; i++) { 264 vd = vq->vq_indirect; 265 vd += vq->vq_maxnsegs * i; 266 for (j = 0; j < vq->vq_maxnsegs-1; j++) 267 vd[j].next = j + 1; 268 } 269 } 270 271 /* free slot management */ 272 SLIST_INIT(&vq->vq_freelist); 273 /* 274 * virtio_enqueue_trim needs monotonely raising entries, therefore 275 * initialize in reverse order 276 */ 277 for (i = vq_size - 1; i >= 0; i--) { 278 SLIST_INSERT_HEAD(&vq->vq_freelist, &vq->vq_entries[i], 279 qe_list); 280 vq->vq_entries[i].qe_index = i; 281 } 282 283 /* enqueue/dequeue status */ 284 vq->vq_avail_idx = 0; 285 vq->vq_used_idx = 0; 286 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 287 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 288 vq->vq_queued = 1; 289} 290 291/* 292 * Allocate/free a vq. 293 * 294 * maxnsegs denotes how much space should be allocated for indirect 295 * descriptors. maxnsegs == 1 can be used to disable use indirect 296 * descriptors for this queue. 297 */ 298int 299virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 300 int maxsegsize, int maxnsegs, const char *name) 301{ 302 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 303 int rsegs, r, hdrlen; 304#define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 305 ~(VIRTIO_PAGE_SIZE-1)) 306 307 memset(vq, 0, sizeof(*vq)); 308 309 vq_size = virtio_read_queue_size(sc, index); 310 if (vq_size == 0) { 311 printf("virtqueue not exist, index %d for %s\n", index, name); 312 goto err; 313 } 314 if (((vq_size - 1) & vq_size) != 0) 315 panic("vq_size not power of two: %d", vq_size); 316 317 hdrlen = virtio_has_feature(sc, VIRTIO_F_RING_EVENT_IDX) ? 3 : 2; 318 319 /* allocsize1: descriptor table + avail ring + pad */ 320 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc) * vq_size 321 + sizeof(uint16_t) * (hdrlen + vq_size)); 322 /* allocsize2: used ring + pad */ 323 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t) * hdrlen 324 + sizeof(struct vring_used_elem) * vq_size); 325 /* allocsize3: indirect table */ 326 if (sc->sc_indirect && maxnsegs > 1) 327 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 328 else 329 allocsize3 = 0; 330 allocsize = allocsize1 + allocsize2 + allocsize3; 331 332 /* alloc and map the memory */ 333 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 334 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 335 if (r != 0) { 336 printf("virtqueue %d for %s allocation failed, error %d\n", 337 index, name, r); 338 goto err; 339 } 340 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 341 (caddr_t*)&vq->vq_vaddr, BUS_DMA_NOWAIT); 342 if (r != 0) { 343 printf("virtqueue %d for %s map failed, error %d\n", index, 344 name, r); 345 goto err; 346 } 347 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 348 BUS_DMA_NOWAIT, &vq->vq_dmamap); 349 if (r != 0) { 350 printf("virtqueue %d for %s dmamap creation failed, " 351 "error %d\n", index, name, r); 352 goto err; 353 } 354 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, vq->vq_vaddr, 355 allocsize, NULL, BUS_DMA_NOWAIT); 356 if (r != 0) { 357 printf("virtqueue %d for %s dmamap load failed, error %d\n", 358 index, name, r); 359 goto err; 360 } 361 362 /* remember addresses and offsets for later use */ 363 vq->vq_owner = sc; 364 vq->vq_num = vq_size; 365 vq->vq_mask = vq_size - 1; 366 vq->vq_index = index; 367 vq->vq_desc = vq->vq_vaddr; 368 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 369 vq->vq_avail = (struct vring_avail*)(((char*)vq->vq_desc) + 370 vq->vq_availoffset); 371 vq->vq_usedoffset = allocsize1; 372 vq->vq_used = (struct vring_used*)(((char*)vq->vq_desc) + 373 vq->vq_usedoffset); 374 if (allocsize3 > 0) { 375 vq->vq_indirectoffset = allocsize1 + allocsize2; 376 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 377 + vq->vq_indirectoffset); 378 } 379 vq->vq_bytesize = allocsize; 380 vq->vq_maxnsegs = maxnsegs; 381 382 /* free slot management */ 383 vq->vq_entries = mallocarray(vq_size, sizeof(struct vq_entry), 384 M_DEVBUF, M_NOWAIT | M_ZERO); 385 if (vq->vq_entries == NULL) { 386 r = ENOMEM; 387 goto err; 388 } 389 390 virtio_init_vq(sc, vq); 391 virtio_setup_queue(sc, vq, vq->vq_dmamap->dm_segs[0].ds_addr); 392 393#if VIRTIO_DEBUG 394 printf("\nallocated %u byte for virtqueue %d for %s, size %d\n", 395 allocsize, index, name, vq_size); 396 if (allocsize3 > 0) 397 printf("using %d byte (%d entries) indirect descriptors\n", 398 allocsize3, maxnsegs * vq_size); 399#endif 400 return 0; 401 402err: 403 if (vq->vq_dmamap) 404 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 405 if (vq->vq_vaddr) 406 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 407 if (vq->vq_segs[0].ds_addr) 408 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 409 memset(vq, 0, sizeof(*vq)); 410 411 return -1; 412} 413 414int 415virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 416{ 417 struct vq_entry *qe; 418 int i = 0; 419 420 /* device must be already deactivated */ 421 /* confirm the vq is empty */ 422 SLIST_FOREACH(qe, &vq->vq_freelist, qe_list) { 423 i++; 424 } 425 if (i != vq->vq_num) { 426 printf("%s: freeing non-empty vq, index %d\n", 427 sc->sc_dev.dv_xname, vq->vq_index); 428 return EBUSY; 429 } 430 431 /* tell device that there's no virtqueue any longer */ 432 virtio_setup_queue(sc, vq, 0); 433 434 free(vq->vq_entries, M_DEVBUF, 0); 435 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 436 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 437 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 438 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 439 memset(vq, 0, sizeof(*vq)); 440 441 return 0; 442} 443 444/* 445 * Free descriptor management. 446 */ 447struct vq_entry * 448vq_alloc_entry(struct virtqueue *vq) 449{ 450 struct vq_entry *qe; 451 452 if (SLIST_EMPTY(&vq->vq_freelist)) 453 return NULL; 454 qe = SLIST_FIRST(&vq->vq_freelist); 455 SLIST_REMOVE_HEAD(&vq->vq_freelist, qe_list); 456 457 return qe; 458} 459 460void 461vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 462{ 463 SLIST_INSERT_HEAD(&vq->vq_freelist, qe, qe_list); 464} 465 466/* 467 * Enqueue several dmamaps as a single request. 468 */ 469/* 470 * Typical usage: 471 * <queue size> number of followings are stored in arrays 472 * - command blocks (in dmamem) should be pre-allocated and mapped 473 * - dmamaps for command blocks should be pre-allocated and loaded 474 * - dmamaps for payload should be pre-allocated 475 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 476 * if (r) // currently 0 or EAGAIN 477 * return r; 478 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 479 * if (r) { 480 * virtio_enqueue_abort(sc, vq, slot); 481 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 482 * return r; 483 * } 484 * r = virtio_enqueue_reserve(sc, vq, slot, 485 * dmamap_payload[slot]->dm_nsegs+1); 486 * // ^ +1 for command 487 * if (r) { // currently 0 or EAGAIN 488 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 489 * return r; // do not call abort() 490 * } 491 * <setup and prepare commands> 492 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 493 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 494 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], 0); 495 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 496 * virtio_enqueue_commit(sc, vq, slot, 1); 497 * 498 * Alternative usage with statically allocated slots: 499 * <during initialization> 500 * // while not out of slots, do 501 * virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 502 * virtio_enqueue_reserve(sc, vq, slot, max_segs); // reserve all slots 503 * that may ever be needed 504 * 505 * <when enqueuing a request> 506 * // Don't call virtio_enqueue_prep() 507 * bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 508 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 509 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 510 * virtio_enqueue_trim(sc, vq, slot, num_segs_needed); 511 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], 0); 512 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 513 * virtio_enqueue_commit(sc, vq, slot, 1); 514 * 515 * <when dequeuing> 516 * // don't call virtio_dequeue_commit() 517 */ 518 519/* 520 * enqueue_prep: allocate a slot number 521 */ 522int 523virtio_enqueue_prep(struct virtqueue *vq, int *slotp) 524{ 525 struct vq_entry *qe1; 526 527 VIRTIO_ASSERT(slotp != NULL); 528 529 qe1 = vq_alloc_entry(vq); 530 if (qe1 == NULL) 531 return EAGAIN; 532 /* next slot is not allocated yet */ 533 qe1->qe_next = -1; 534 *slotp = qe1->qe_index; 535 536 return 0; 537} 538 539/* 540 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 541 * Calls virtio_enqueue_abort() on failure. 542 */ 543int 544virtio_enqueue_reserve(struct virtqueue *vq, int slot, int nsegs) 545{ 546 struct vq_entry *qe1 = &vq->vq_entries[slot]; 547 548 VIRTIO_ASSERT(qe1->qe_next == -1); 549 VIRTIO_ASSERT(1 <= nsegs && nsegs <= vq->vq_num); 550 551 if (vq->vq_indirect != NULL && nsegs > 1 && nsegs <= vq->vq_maxnsegs) { 552 struct vring_desc *vd; 553 int i; 554 555 qe1->qe_indirect = 1; 556 557 vd = &vq->vq_desc[qe1->qe_index]; 558 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr + 559 vq->vq_indirectoffset; 560 vd->addr += sizeof(struct vring_desc) * vq->vq_maxnsegs * 561 qe1->qe_index; 562 vd->len = sizeof(struct vring_desc) * nsegs; 563 vd->flags = VRING_DESC_F_INDIRECT; 564 565 vd = vq->vq_indirect; 566 vd += vq->vq_maxnsegs * qe1->qe_index; 567 qe1->qe_desc_base = vd; 568 569 for (i = 0; i < nsegs-1; i++) 570 vd[i].flags = VRING_DESC_F_NEXT; 571 vd[i].flags = 0; 572 qe1->qe_next = 0; 573 574 return 0; 575 } else { 576 struct vring_desc *vd; 577 struct vq_entry *qe; 578 int i, s; 579 580 qe1->qe_indirect = 0; 581 582 vd = &vq->vq_desc[0]; 583 qe1->qe_desc_base = vd; 584 qe1->qe_next = qe1->qe_index; 585 s = slot; 586 for (i = 0; i < nsegs - 1; i++) { 587 qe = vq_alloc_entry(vq); 588 if (qe == NULL) { 589 vd[s].flags = 0; 590 virtio_enqueue_abort(vq, slot); 591 return EAGAIN; 592 } 593 vd[s].flags = VRING_DESC_F_NEXT; 594 vd[s].next = qe->qe_index; 595 s = qe->qe_index; 596 } 597 vd[s].flags = 0; 598 599 return 0; 600 } 601} 602 603/* 604 * enqueue: enqueue a single dmamap. 605 */ 606int 607virtio_enqueue(struct virtqueue *vq, int slot, bus_dmamap_t dmamap, int write) 608{ 609 struct vq_entry *qe1 = &vq->vq_entries[slot]; 610 struct vring_desc *vd = qe1->qe_desc_base; 611 int i; 612 int s = qe1->qe_next; 613 614 VIRTIO_ASSERT(s >= 0); 615 VIRTIO_ASSERT(dmamap->dm_nsegs > 0); 616 if (dmamap->dm_nsegs > vq->vq_maxnsegs) { 617#if VIRTIO_DEBUG 618 for (i = 0; i < dmamap->dm_nsegs; i++) { 619 printf(" %d (%d): %p %lx \n", i, write, 620 (void *)dmamap->dm_segs[i].ds_addr, 621 dmamap->dm_segs[i].ds_len); 622 } 623#endif 624 panic("dmamap->dm_nseg %d > vq->vq_maxnsegs %d", 625 dmamap->dm_nsegs, vq->vq_maxnsegs); 626 } 627 628 for (i = 0; i < dmamap->dm_nsegs; i++) { 629 vd[s].addr = dmamap->dm_segs[i].ds_addr; 630 vd[s].len = dmamap->dm_segs[i].ds_len; 631 if (!write) 632 vd[s].flags |= VRING_DESC_F_WRITE; 633 s = vd[s].next; 634 } 635 qe1->qe_next = s; 636 637 return 0; 638} 639 640int 641virtio_enqueue_p(struct virtqueue *vq, int slot, bus_dmamap_t dmamap, 642 bus_addr_t start, bus_size_t len, int write) 643{ 644 struct vq_entry *qe1 = &vq->vq_entries[slot]; 645 struct vring_desc *vd = qe1->qe_desc_base; 646 int s = qe1->qe_next; 647 648 VIRTIO_ASSERT(s >= 0); 649 /* XXX todo: handle more segments */ 650 VIRTIO_ASSERT(dmamap->dm_nsegs == 1); 651 VIRTIO_ASSERT((dmamap->dm_segs[0].ds_len > start) && 652 (dmamap->dm_segs[0].ds_len >= start + len)); 653 654 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 655 vd[s].len = len; 656 if (!write) 657 vd[s].flags |= VRING_DESC_F_WRITE; 658 qe1->qe_next = vd[s].next; 659 660 return 0; 661} 662 663static void 664publish_avail_idx(struct virtio_softc *sc, struct virtqueue *vq) 665{ 666 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 667 668 virtio_membar_producer(); 669 vq->vq_avail->idx = vq->vq_avail_idx; 670 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 671 vq->vq_queued = 1; 672} 673 674/* 675 * enqueue_commit: add it to the aring. 676 */ 677void 678virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 679 int notifynow) 680{ 681 struct vq_entry *qe1; 682 683 if (slot < 0) 684 goto notify; 685 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 686 qe1 = &vq->vq_entries[slot]; 687 if (qe1->qe_indirect) 688 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 689 vq->vq_avail->ring[(vq->vq_avail_idx++) & vq->vq_mask] = slot; 690 691notify: 692 if (notifynow) { 693 if (virtio_has_feature(vq->vq_owner, VIRTIO_F_RING_EVENT_IDX)) { 694 uint16_t o = vq->vq_avail->idx; 695 uint16_t n = vq->vq_avail_idx; 696 uint16_t t; 697 publish_avail_idx(sc, vq); 698 699 virtio_membar_sync(); 700 t = VQ_AVAIL_EVENT(vq) + 1; 701 if ((uint16_t)(n - t) < (uint16_t)(n - o)) 702 sc->sc_ops->kick(sc, vq->vq_index); 703 } else { 704 publish_avail_idx(sc, vq); 705 706 virtio_membar_sync(); 707 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 708 sc->sc_ops->kick(sc, vq->vq_index); 709 } 710 } 711} 712 713/* 714 * enqueue_abort: rollback. 715 */ 716int 717virtio_enqueue_abort(struct virtqueue *vq, int slot) 718{ 719 struct vq_entry *qe = &vq->vq_entries[slot]; 720 struct vring_desc *vd; 721 int s; 722 723 if (qe->qe_next < 0) { 724 vq_free_entry(vq, qe); 725 return 0; 726 } 727 728 s = slot; 729 vd = &vq->vq_desc[0]; 730 while (vd[s].flags & VRING_DESC_F_NEXT) { 731 s = vd[s].next; 732 vq_free_entry(vq, qe); 733 qe = &vq->vq_entries[s]; 734 } 735 vq_free_entry(vq, qe); 736 return 0; 737} 738 739/* 740 * enqueue_trim: adjust buffer size to given # of segments, a.k.a. 741 * descriptors. 742 */ 743void 744virtio_enqueue_trim(struct virtqueue *vq, int slot, int nsegs) 745{ 746 struct vq_entry *qe1 = &vq->vq_entries[slot]; 747 struct vring_desc *vd = &vq->vq_desc[0]; 748 int i; 749 750 if ((vd[slot].flags & VRING_DESC_F_INDIRECT) == 0) { 751 qe1->qe_next = qe1->qe_index; 752 /* 753 * N.B.: the vq_entries are ASSUMED to be a contiguous 754 * block with slot being the index to the first one. 755 */ 756 } else { 757 qe1->qe_next = 0; 758 vd = &vq->vq_desc[qe1->qe_index]; 759 vd->len = sizeof(struct vring_desc) * nsegs; 760 vd = qe1->qe_desc_base; 761 slot = 0; 762 } 763 764 for (i = 0; i < nsegs -1 ; i++) { 765 vd[slot].flags = VRING_DESC_F_NEXT; 766 slot++; 767 } 768 vd[slot].flags = 0; 769} 770 771/* 772 * Dequeue a request. 773 */ 774/* 775 * dequeue: dequeue a request from uring; dmamap_sync for uring is 776 * already done in the interrupt handler. 777 */ 778int 779virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 780 int *slotp, int *lenp) 781{ 782 uint16_t slot, usedidx; 783 struct vq_entry *qe; 784 785 if (vq->vq_used_idx == vq->vq_used->idx) 786 return ENOENT; 787 usedidx = vq->vq_used_idx++; 788 usedidx &= vq->vq_mask; 789 790 virtio_membar_consumer(); 791 slot = vq->vq_used->ring[usedidx].id; 792 qe = &vq->vq_entries[slot]; 793 794 if (qe->qe_indirect) 795 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 796 797 if (slotp) 798 *slotp = slot; 799 if (lenp) 800 *lenp = vq->vq_used->ring[usedidx].len; 801 802 return 0; 803} 804 805/* 806 * dequeue_commit: complete dequeue; the slot is recycled for future use. 807 * if you forget to call this the slot will be leaked. 808 * 809 * Don't call this if you use statically allocated slots 810 * and virtio_dequeue_trim(). 811 */ 812int 813virtio_dequeue_commit(struct virtqueue *vq, int slot) 814{ 815 struct vq_entry *qe = &vq->vq_entries[slot]; 816 struct vring_desc *vd = &vq->vq_desc[0]; 817 int s = slot; 818 819 while (vd[s].flags & VRING_DESC_F_NEXT) { 820 s = vd[s].next; 821 vq_free_entry(vq, qe); 822 qe = &vq->vq_entries[s]; 823 } 824 vq_free_entry(vq, qe); 825 826 return 0; 827} 828 829/* 830 * Increase the event index in order to delay interrupts. 831 * Returns 0 on success; returns 1 if the used ring has already advanced 832 * too far, and the caller must process the queue again (otherwise, no 833 * more interrupts will happen). 834 */ 835int 836virtio_postpone_intr(struct virtqueue *vq, uint16_t nslots) 837{ 838 uint16_t idx; 839 840 idx = vq->vq_used_idx + nslots; 841 842 /* set the new event index: avail_ring->used_event = idx */ 843 VQ_USED_EVENT(vq) = idx; 844 virtio_membar_sync(); 845 846 vq_sync_aring(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 847 vq->vq_queued++; 848 849 if (nslots < virtio_nused(vq)) 850 return 1; 851 852 return 0; 853} 854 855/* 856 * Postpone interrupt until 3/4 of the available descriptors have been 857 * consumed. 858 */ 859int 860virtio_postpone_intr_smart(struct virtqueue *vq) 861{ 862 uint16_t nslots; 863 864 nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx) * 3 / 4; 865 866 return virtio_postpone_intr(vq, nslots); 867} 868 869/* 870 * Postpone interrupt until all of the available descriptors have been 871 * consumed. 872 */ 873int 874virtio_postpone_intr_far(struct virtqueue *vq) 875{ 876 uint16_t nslots; 877 878 nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx); 879 880 return virtio_postpone_intr(vq, nslots); 881} 882 883 884/* 885 * Start/stop vq interrupt. No guarantee. 886 */ 887void 888virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 889{ 890 if (virtio_has_feature(sc, VIRTIO_F_RING_EVENT_IDX)) { 891 /* 892 * No way to disable the interrupt completely with 893 * RingEventIdx. Instead advance used_event by half 894 * the possible value. This won't happen soon and 895 * is far enough in the past to not trigger a spurious 896 * interrupt. 897 */ 898 VQ_USED_EVENT(vq) = vq->vq_used_idx + 0x8000; 899 } else { 900 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 901 } 902 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 903 vq->vq_queued++; 904} 905 906int 907virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 908{ 909 /* 910 * If event index feature is negotiated, enabling 911 * interrupts is done through setting the latest 912 * consumed index in the used_event field 913 */ 914 if (virtio_has_feature(sc, VIRTIO_F_RING_EVENT_IDX)) 915 VQ_USED_EVENT(vq) = vq->vq_used_idx; 916 else 917 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 918 919 virtio_membar_sync(); 920 921 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 922 vq->vq_queued++; 923 924 if (vq->vq_used_idx != vq->vq_used->idx) 925 return 1; 926 927 return 0; 928} 929 930/* 931 * Returns a number of slots in the used ring available to 932 * be supplied to the avail ring. 933 */ 934int 935virtio_nused(struct virtqueue *vq) 936{ 937 uint16_t n; 938 939 n = (uint16_t)(vq->vq_used->idx - vq->vq_used_idx); 940 VIRTIO_ASSERT(n <= vq->vq_num); 941 942 return n; 943} 944 945#if VIRTIO_DEBUG 946void 947virtio_vq_dump(struct virtqueue *vq) 948{ 949 /* Common fields */ 950 printf(" + vq num: %d\n", vq->vq_num); 951 printf(" + vq mask: 0x%X\n", vq->vq_mask); 952 printf(" + vq index: %d\n", vq->vq_index); 953 printf(" + vq used idx: %d\n", vq->vq_used_idx); 954 printf(" + vq avail idx: %d\n", vq->vq_avail_idx); 955 printf(" + vq queued: %d\n",vq->vq_queued); 956 /* Avail ring fields */ 957 printf(" + avail flags: 0x%X\n", vq->vq_avail->flags); 958 printf(" + avail idx: %d\n", vq->vq_avail->idx); 959 printf(" + avail event: %d\n", VQ_AVAIL_EVENT(vq)); 960 /* Used ring fields */ 961 printf(" + used flags: 0x%X\n",vq->vq_used->flags); 962 printf(" + used idx: %d\n",vq->vq_used->idx); 963 printf(" + used event: %d\n", VQ_USED_EVENT(vq)); 964 printf(" +++++++++++++++++++++++++++\n"); 965} 966#endif 967