virtio.c revision 1.21
1/* $NetBSD: virtio.c,v 1.21 2017/03/25 17:50:51 jdolecek Exp $ */ 2 3/* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.21 2017/03/25 17:50:51 jdolecek Exp $"); 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/kernel.h> 34#include <sys/atomic.h> 35#include <sys/bus.h> 36#include <sys/device.h> 37#include <sys/kmem.h> 38#include <sys/module.h> 39 40#include <dev/pci/pcidevs.h> 41#include <dev/pci/pcireg.h> 42#include <dev/pci/pcivar.h> 43 44#include <dev/pci/virtioreg.h> 45#include <dev/pci/virtiovar.h> 46 47#define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49static int virtio_match(device_t, cfdata_t, void *); 50static void virtio_attach(device_t, device_t, void *); 51static int virtio_rescan(device_t, const char *, const int *); 52static int virtio_detach(device_t, int); 53static int virtio_intr(void *arg); 54static int virtio_msix_queue_intr(void *); 55static int virtio_msix_config_intr(void *); 56static int virtio_setup_msix_vectors(struct virtio_softc *); 57static int virtio_setup_msix_interrupts(struct virtio_softc *, 58 struct pci_attach_args *); 59static int virtio_setup_intx_interrupt(struct virtio_softc *, 60 struct pci_attach_args *); 61static int virtio_setup_interrupts(struct virtio_softc *); 62static void virtio_soft_intr(void *arg); 63static void virtio_init_vq(struct virtio_softc *, 64 struct virtqueue *, const bool); 65 66CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 67 virtio_match, virtio_attach, virtio_detach, NULL, virtio_rescan, NULL, 68 DVF_DETACH_SHUTDOWN); 69 70/* we use the legacy virtio spec, so the pci registers are host native 71 * byte order, not pci (i.e. LE) byte order */ 72static inline uint16_t 73nbo_bus_space_read_2(bus_space_tag_t space, bus_space_handle_t handle, 74 bus_size_t offset) 75{ 76 return le16toh(bus_space_read_2(space, handle, offset)); 77} 78 79static inline uint32_t 80nbo_bus_space_read_4(bus_space_tag_t space, bus_space_handle_t handle, 81 bus_size_t offset) 82{ 83 return le32toh(bus_space_read_4(space, handle, offset)); 84} 85 86static void 87nbo_bus_space_write_2(bus_space_tag_t space, bus_space_handle_t handle, 88 bus_size_t offset, uint16_t value) 89{ 90 bus_space_write_2(space, handle, offset, htole16(value)); 91} 92 93static void 94nbo_bus_space_write_4(bus_space_tag_t space, bus_space_handle_t handle, 95 bus_size_t offset, uint16_t value) 96{ 97 bus_space_write_4(space, handle, offset, htole32(value)); 98} 99 100/* some functions access registers at 4 byte offset for little/high halves */ 101#if BYTE_ORDER == BIG_ENDIAN 102#define REG_HI_OFF 0 103#define REG_LO_OFF 4 104#else 105#define REG_HI_OFF 4 106#define REG_LO_OFF 0 107#endif 108 109static void 110virtio_set_status(struct virtio_softc *sc, int status) 111{ 112 int old = 0; 113 114 if (status != 0) 115 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 116 VIRTIO_CONFIG_DEVICE_STATUS); 117 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 118 status|old); 119} 120 121#define virtio_device_reset(sc) virtio_set_status((sc), 0) 122 123static int 124virtio_match(device_t parent, cfdata_t match, void *aux) 125{ 126 struct pci_attach_args *pa; 127 128 pa = (struct pci_attach_args *)aux; 129 switch (PCI_VENDOR(pa->pa_id)) { 130 case PCI_VENDOR_QUMRANET: 131 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 132 PCI_PRODUCT(pa->pa_id)) && 133 (PCI_PRODUCT(pa->pa_id) <= 134 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 135 return 1; 136 break; 137 } 138 139 return 0; 140} 141 142static const char *virtio_device_name[] = { 143 "Unknown (0)", /* 0 */ 144 "Network", /* 1 */ 145 "Block", /* 2 */ 146 "Console", /* 3 */ 147 "Entropy", /* 4 */ 148 "Memory Balloon", /* 5 */ 149 "I/O Memory", /* 6 */ 150 "Remote Processor Messaging", /* 7 */ 151 "SCSI", /* 8 */ 152 "9P Transport", /* 9 */ 153 "mac80211 wlan", /* 10 */ 154}; 155#define NDEVNAMES __arraycount(virtio_device_name) 156 157#define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 158#define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 159 160static int 161virtio_setup_msix_vectors(struct virtio_softc *sc) 162{ 163 int offset, vector, ret, qid; 164 165 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 166 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 167 168 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 169 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 170 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 171 vector, ret); 172 if (ret != vector) 173 return -1; 174 175 for (qid = 0; qid < sc->sc_nvqs; qid++) { 176 offset = VIRTIO_CONFIG_QUEUE_SELECT; 177 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 178 179 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 180 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 181 182 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 183 ret = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 184 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 185 vector, ret); 186 if (ret != vector) 187 return -1; 188 } 189 190 return 0; 191} 192 193static int 194virtio_setup_msix_interrupts(struct virtio_softc *sc, 195 struct pci_attach_args *pa) 196{ 197 device_t self = sc->sc_dev; 198 pci_chipset_tag_t pc = pa->pa_pc; 199 char intrbuf[PCI_INTRSTR_LEN]; 200 char const *intrstr; 201 int idx; 202 203 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 204 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 205 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 206 207 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 208 sc->sc_ipl, virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 209 if (sc->sc_ihs[idx] == NULL) { 210 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 211 goto error; 212 } 213 214 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 215 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 216 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 217 218 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], 219 sc->sc_ipl, virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 220 if (sc->sc_ihs[idx] == NULL) { 221 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 222 goto error; 223 } 224 225 if (virtio_setup_msix_vectors(sc) != 0) { 226 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 227 goto error; 228 } 229 230 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 231 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 232 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 233 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 234 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 235 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 236 237 return 0; 238 239error: 240 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 241 if (sc->sc_ihs[idx] != NULL) 242 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 243 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 244 if (sc->sc_ihs[idx] != NULL) 245 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 246 247 return -1; 248} 249 250static int 251virtio_setup_intx_interrupt(struct virtio_softc *sc, 252 struct pci_attach_args *pa) 253{ 254 device_t self = sc->sc_dev; 255 pci_chipset_tag_t pc = pa->pa_pc; 256 char intrbuf[PCI_INTRSTR_LEN]; 257 char const *intrstr; 258 259 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 260 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 261 262 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 263 sc->sc_ipl, virtio_intr, sc, device_xname(sc->sc_dev)); 264 if (sc->sc_ihs[0] == NULL) { 265 aprint_error_dev(self, "couldn't establish INTx\n"); 266 return -1; 267 } 268 269 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 270 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 271 272 return 0; 273} 274 275static int 276virtio_setup_interrupts(struct virtio_softc *sc) 277{ 278 device_t self = sc->sc_dev; 279 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 280 int error; 281 int nmsix; 282 int counts[PCI_INTR_TYPE_SIZE]; 283 pci_intr_type_t max_type; 284 285 nmsix = pci_msix_count(sc->sc_pa.pa_pc, sc->sc_pa.pa_tag); 286 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 287 288 /* We need at least two: one for config and the other for queues */ 289 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 290 /* Try INTx only */ 291 max_type = PCI_INTR_TYPE_INTX; 292 counts[PCI_INTR_TYPE_INTX] = 1; 293 } else { 294 /* Try MSI-X first and INTx second */ 295 max_type = PCI_INTR_TYPE_MSIX; 296 counts[PCI_INTR_TYPE_MSIX] = 2; 297 counts[PCI_INTR_TYPE_MSI] = 0; 298 counts[PCI_INTR_TYPE_INTX] = 1; 299 } 300 301 retry: 302 error = pci_intr_alloc(&sc->sc_pa, &sc->sc_ihp, counts, max_type); 303 if (error != 0) { 304 aprint_error_dev(self, "couldn't map interrupt\n"); 305 return -1; 306 } 307 308 if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 309 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, 310 KM_SLEEP); 311 if (sc->sc_ihs == NULL) { 312 pci_intr_release(pc, sc->sc_ihp, 2); 313 314 /* Retry INTx */ 315 max_type = PCI_INTR_TYPE_INTX; 316 counts[PCI_INTR_TYPE_INTX] = 1; 317 goto retry; 318 } 319 320 error = virtio_setup_msix_interrupts(sc, &sc->sc_pa); 321 if (error != 0) { 322 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 323 pci_intr_release(pc, sc->sc_ihp, 2); 324 325 /* Retry INTx */ 326 max_type = PCI_INTR_TYPE_INTX; 327 counts[PCI_INTR_TYPE_INTX] = 1; 328 goto retry; 329 } 330 331 sc->sc_ihs_num = 2; 332 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 333 } else if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 334 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, 335 KM_SLEEP); 336 if (sc->sc_ihs == NULL) { 337 pci_intr_release(pc, sc->sc_ihp, 1); 338 return -1; 339 } 340 341 error = virtio_setup_intx_interrupt(sc, &sc->sc_pa); 342 if (error != 0) { 343 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 344 pci_intr_release(pc, sc->sc_ihp, 1); 345 return -1; 346 } 347 348 sc->sc_ihs_num = 1; 349 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 350 } 351 352 return 0; 353} 354 355static void 356virtio_attach(device_t parent, device_t self, void *aux) 357{ 358 struct virtio_softc *sc = device_private(self); 359 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 360 pci_chipset_tag_t pc = pa->pa_pc; 361 pcitag_t tag = pa->pa_tag; 362 int revision; 363 pcireg_t id; 364 365 revision = PCI_REVISION(pa->pa_class); 366 if (revision != 0) { 367 aprint_normal(": unknown revision 0x%02x; giving up\n", 368 revision); 369 return; 370 } 371 aprint_normal("\n"); 372 aprint_naive("\n"); 373 374 /* subsystem ID shows what I am */ 375 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 376 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 377 (PCI_SUBSYS_ID(id) < NDEVNAMES? 378 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 379 revision); 380 381 sc->sc_dev = self; 382 sc->sc_pc = pc; 383 sc->sc_tag = tag; 384 sc->sc_iot = pa->pa_iot; 385 if (pci_dma64_available(pa)) 386 sc->sc_dmat = pa->pa_dmat64; 387 else 388 sc->sc_dmat = pa->pa_dmat; 389 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 390 391 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 392 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 393 aprint_error_dev(self, "can't map i/o space\n"); 394 return; 395 } 396 397 virtio_device_reset(sc); 398 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 399 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 400 401 /* XXX: use softc as aux... */ 402 sc->sc_childdevid = PCI_SUBSYS_ID(id); 403 sc->sc_child = NULL; 404 sc->sc_pa = *pa; 405 virtio_rescan(self, "virtio", 0); 406 return; 407} 408 409/* ARGSUSED */ 410static int 411virtio_rescan(device_t self, const char *attr, const int *scan_flags) 412{ 413 struct virtio_softc *sc; 414 int r; 415 416 sc = device_private(self); 417 if (sc->sc_child) /* Child already attached? */ 418 return 0; 419 config_found_ia(self, attr, sc, NULL); 420 if (sc->sc_child == NULL) { 421 aprint_error_dev(self, 422 "no matching child driver; not configured\n"); 423 return 0; 424 } 425 if (sc->sc_child == (void*)1) { /* this shows error */ 426 aprint_error_dev(self, 427 "virtio configuration failed\n"); 428 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 429 return 0; 430 } 431 432 r = virtio_setup_interrupts(sc); 433 if (r != 0) { 434 aprint_error_dev(self, "failed to setup interrupts\n"); 435 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 436 return 0; 437 } 438 439 sc->sc_soft_ih = NULL; 440 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 441 u_int flags = SOFTINT_NET; 442 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 443 flags |= SOFTINT_MPSAFE; 444 445 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 446 if (sc->sc_soft_ih == NULL) 447 aprint_error(": failed to establish soft interrupt\n"); 448 } 449 450 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 451 452 return 0; 453} 454 455static int 456virtio_detach(device_t self, int flags) 457{ 458 struct virtio_softc *sc = device_private(self); 459 int r; 460 int i; 461 462 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 463 r = config_detach(sc->sc_child, flags); 464 if (r) 465 return r; 466 } 467 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 468 KASSERT(sc->sc_vqs == 0); 469 for (i = 0; i < sc->sc_ihs_num; i++) { 470 if (sc->sc_ihs[i] == NULL) 471 continue; 472 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 473 } 474 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 475 if (sc->sc_ihs != NULL) 476 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 477 sc->sc_ihs_num = 0; 478 if (sc->sc_iosize) 479 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 480 sc->sc_iosize = 0; 481 482 return 0; 483} 484 485/* 486 * Reset the device. 487 */ 488/* 489 * To reset the device to a known state, do following: 490 * virtio_reset(sc); // this will stop the device activity 491 * <dequeue finished requests>; // virtio_dequeue() still can be called 492 * <revoke pending requests in the vqs if any>; 493 * virtio_reinit_begin(sc); // dequeue prohibitted 494 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 495 * <some other initialization>; 496 * virtio_reinit_end(sc); // device activated; enqueue allowed 497 * Once attached, feature negotiation can only be allowed after virtio_reset. 498 */ 499void 500virtio_reset(struct virtio_softc *sc) 501{ 502 virtio_device_reset(sc); 503} 504 505void 506virtio_reinit_start(struct virtio_softc *sc) 507{ 508 int i; 509 510 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 511 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 512 for (i = 0; i < sc->sc_nvqs; i++) { 513 int n; 514 struct virtqueue *vq = &sc->sc_vqs[i]; 515 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 516 VIRTIO_CONFIG_QUEUE_SELECT, 517 vq->vq_index); 518 n = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 519 VIRTIO_CONFIG_QUEUE_SIZE); 520 if (n == 0) /* vq disappeared */ 521 continue; 522 if (n != vq->vq_num) { 523 panic("%s: virtqueue size changed, vq index %d\n", 524 device_xname(sc->sc_dev), 525 vq->vq_index); 526 } 527 virtio_init_vq(sc, vq, true); 528 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 529 VIRTIO_CONFIG_QUEUE_ADDRESS, 530 (vq->vq_dmamap->dm_segs[0].ds_addr 531 / VIRTIO_PAGE_SIZE)); 532 } 533 534 /* MSI-X should have more than one handles where INTx has just one */ 535 if (sc->sc_ihs_num > 1) { 536 if (virtio_setup_msix_vectors(sc) != 0) { 537 aprint_error_dev(sc->sc_dev, 538 "couldn't setup MSI-X vectors\n"); 539 return; 540 } 541 } 542} 543 544void 545virtio_reinit_end(struct virtio_softc *sc) 546{ 547 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 548} 549 550/* 551 * Feature negotiation. 552 */ 553uint32_t 554virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 555{ 556 uint32_t r; 557 558 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 559 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 560 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 561 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 562 VIRTIO_CONFIG_DEVICE_FEATURES); 563 r &= guest_features; 564 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 565 VIRTIO_CONFIG_GUEST_FEATURES, r); 566 sc->sc_features = r; 567 if (r & VIRTIO_F_RING_INDIRECT_DESC) 568 sc->sc_indirect = true; 569 else 570 sc->sc_indirect = false; 571 572 return r; 573} 574 575/* 576 * Device configuration registers. 577 */ 578uint8_t 579virtio_read_device_config_1(struct virtio_softc *sc, int index) 580{ 581 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 582 sc->sc_config_offset + index); 583} 584 585uint16_t 586virtio_read_device_config_2(struct virtio_softc *sc, int index) 587{ 588 return nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 589 sc->sc_config_offset + index); 590} 591 592uint32_t 593virtio_read_device_config_4(struct virtio_softc *sc, int index) 594{ 595 return nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 596 sc->sc_config_offset + index); 597} 598 599uint64_t 600virtio_read_device_config_8(struct virtio_softc *sc, int index) 601{ 602 uint64_t r; 603 604 r = nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 605 sc->sc_config_offset + index + REG_HI_OFF); 606 r <<= 32; 607 r |= nbo_bus_space_read_4(sc->sc_iot, sc->sc_ioh, 608 sc->sc_config_offset + index + REG_LO_OFF); 609 610 return r; 611} 612 613void 614virtio_write_device_config_1(struct virtio_softc *sc, 615 int index, uint8_t value) 616{ 617 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 618 sc->sc_config_offset + index, value); 619} 620 621void 622virtio_write_device_config_2(struct virtio_softc *sc, 623 int index, uint16_t value) 624{ 625 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 626 sc->sc_config_offset + index, value); 627} 628 629void 630virtio_write_device_config_4(struct virtio_softc *sc, 631 int index, uint32_t value) 632{ 633 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 634 sc->sc_config_offset + index, value); 635} 636 637void 638virtio_write_device_config_8(struct virtio_softc *sc, 639 int index, uint64_t value) 640{ 641 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 642 sc->sc_config_offset + index + REG_LO_OFF, 643 value & 0xffffffff); 644 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 645 sc->sc_config_offset + index + REG_HI_OFF, 646 value >> 32); 647} 648 649/* 650 * Interrupt handler. 651 */ 652static int 653virtio_intr(void *arg) 654{ 655 struct virtio_softc *sc = arg; 656 int isr, r = 0; 657 658 /* check and ack the interrupt */ 659 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 660 VIRTIO_CONFIG_ISR_STATUS); 661 if (isr == 0) 662 return 0; 663 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 664 (sc->sc_config_change != NULL)) 665 r = (sc->sc_config_change)(sc); 666 if (sc->sc_intrhand != NULL) { 667 if (sc->sc_soft_ih != NULL) 668 softint_schedule(sc->sc_soft_ih); 669 else 670 r |= (sc->sc_intrhand)(sc); 671 } 672 673 return r; 674} 675 676static int 677virtio_msix_queue_intr(void *arg) 678{ 679 struct virtio_softc *sc = arg; 680 int r = 0; 681 682 if (sc->sc_intrhand != NULL) { 683 if (sc->sc_soft_ih != NULL) 684 softint_schedule(sc->sc_soft_ih); 685 else 686 r |= (sc->sc_intrhand)(sc); 687 } 688 689 return r; 690} 691 692static int 693virtio_msix_config_intr(void *arg) 694{ 695 struct virtio_softc *sc = arg; 696 697 /* TODO: handle events */ 698 aprint_debug_dev(sc->sc_dev, "%s\n", __func__); 699 return 1; 700} 701 702static void 703virtio_soft_intr(void *arg) 704{ 705 struct virtio_softc *sc = arg; 706 707 KASSERT(sc->sc_intrhand != NULL); 708 709 (sc->sc_intrhand)(sc); 710} 711 712/* 713 * dmamap sync operations for a virtqueue. 714 */ 715static inline void 716vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 717{ 718 /* availoffset == sizeof(vring_desc)*vq_num */ 719 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 720 ops); 721} 722 723static inline void 724vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 725{ 726 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 727 vq->vq_availoffset, 728 offsetof(struct vring_avail, ring) 729 + vq->vq_num * sizeof(uint16_t), 730 ops); 731} 732 733static inline void 734vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 735{ 736 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 737 vq->vq_usedoffset, 738 offsetof(struct vring_used, ring) 739 + vq->vq_num * sizeof(struct vring_used_elem), 740 ops); 741} 742 743static inline void 744vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 745 int ops) 746{ 747 int offset = vq->vq_indirectoffset 748 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 749 750 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 751 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 752 ops); 753} 754 755/* 756 * Can be used as sc_intrhand. 757 */ 758/* 759 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 760 * and calls (*vq_done)() if some entries are consumed. 761 */ 762int 763virtio_vq_intr(struct virtio_softc *sc) 764{ 765 struct virtqueue *vq; 766 int i, r = 0; 767 768 for (i = 0; i < sc->sc_nvqs; i++) { 769 vq = &sc->sc_vqs[i]; 770 if (vq->vq_queued) { 771 vq->vq_queued = 0; 772 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 773 } 774 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 775 membar_consumer(); 776 if (vq->vq_used_idx != vq->vq_used->idx) { 777 if (vq->vq_done) 778 r |= (vq->vq_done)(vq); 779 } 780 } 781 782 return r; 783} 784 785/* 786 * Start/stop vq interrupt. No guarantee. 787 */ 788void 789virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 790{ 791 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 792 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 793 vq->vq_queued++; 794} 795 796void 797virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 798{ 799 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 800 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 801 vq->vq_queued++; 802} 803 804/* 805 * Initialize vq structure. 806 */ 807static void 808virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 809 const bool reinit) 810{ 811 int i, j; 812 int vq_size = vq->vq_num; 813 814 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 815 816 /* build the indirect descriptor chain */ 817 if (vq->vq_indirect != NULL) { 818 struct vring_desc *vd; 819 820 for (i = 0; i < vq_size; i++) { 821 vd = vq->vq_indirect; 822 vd += vq->vq_maxnsegs * i; 823 for (j = 0; j < vq->vq_maxnsegs-1; j++) 824 vd[j].next = j + 1; 825 } 826 } 827 828 /* free slot management */ 829 SIMPLEQ_INIT(&vq->vq_freelist); 830 for (i = 0; i < vq_size; i++) { 831 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 832 &vq->vq_entries[i], qe_list); 833 vq->vq_entries[i].qe_index = i; 834 } 835 if (!reinit) 836 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 837 838 /* enqueue/dequeue status */ 839 vq->vq_avail_idx = 0; 840 vq->vq_used_idx = 0; 841 vq->vq_queued = 0; 842 if (!reinit) { 843 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 844 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 845 } 846 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 847 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 848 vq->vq_queued++; 849} 850 851/* 852 * Allocate/free a vq. 853 */ 854int 855virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 856 int maxsegsize, int maxnsegs, const char *name) 857{ 858 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 859 int rsegs, r; 860#define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 861 ~(VIRTIO_PAGE_SIZE-1)) 862 863 memset(vq, 0, sizeof(*vq)); 864 865 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 866 VIRTIO_CONFIG_QUEUE_SELECT, index); 867 vq_size = nbo_bus_space_read_2(sc->sc_iot, sc->sc_ioh, 868 VIRTIO_CONFIG_QUEUE_SIZE); 869 if (vq_size == 0) { 870 aprint_error_dev(sc->sc_dev, 871 "virtqueue not exist, index %d for %s\n", 872 index, name); 873 goto err; 874 } 875 /* allocsize1: descriptor table + avail ring + pad */ 876 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 877 + sizeof(uint16_t)*(2+vq_size)); 878 /* allocsize2: used ring + pad */ 879 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 880 + sizeof(struct vring_used_elem)*vq_size); 881 /* allocsize3: indirect table */ 882 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 883 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 884 else 885 allocsize3 = 0; 886 allocsize = allocsize1 + allocsize2 + allocsize3; 887 888 /* alloc and map the memory */ 889 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 890 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 891 if (r != 0) { 892 aprint_error_dev(sc->sc_dev, 893 "virtqueue %d for %s allocation failed, " 894 "error code %d\n", index, name, r); 895 goto err; 896 } 897 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 898 &vq->vq_vaddr, BUS_DMA_NOWAIT); 899 if (r != 0) { 900 aprint_error_dev(sc->sc_dev, 901 "virtqueue %d for %s map failed, " 902 "error code %d\n", index, name, r); 903 goto err; 904 } 905 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 906 BUS_DMA_NOWAIT, &vq->vq_dmamap); 907 if (r != 0) { 908 aprint_error_dev(sc->sc_dev, 909 "virtqueue %d for %s dmamap creation failed, " 910 "error code %d\n", index, name, r); 911 goto err; 912 } 913 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 914 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 915 if (r != 0) { 916 aprint_error_dev(sc->sc_dev, 917 "virtqueue %d for %s dmamap load failed, " 918 "error code %d\n", index, name, r); 919 goto err; 920 } 921 922 /* set the vq address */ 923 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 924 VIRTIO_CONFIG_QUEUE_ADDRESS, 925 (vq->vq_dmamap->dm_segs[0].ds_addr 926 / VIRTIO_PAGE_SIZE)); 927 928 /* remember addresses and offsets for later use */ 929 vq->vq_owner = sc; 930 vq->vq_num = vq_size; 931 vq->vq_index = index; 932 vq->vq_desc = vq->vq_vaddr; 933 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 934 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 935 vq->vq_usedoffset = allocsize1; 936 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 937 if (allocsize3 > 0) { 938 vq->vq_indirectoffset = allocsize1 + allocsize2; 939 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 940 + vq->vq_indirectoffset); 941 } 942 vq->vq_bytesize = allocsize; 943 vq->vq_maxsegsize = maxsegsize; 944 vq->vq_maxnsegs = maxnsegs; 945 946 /* free slot management */ 947 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 948 KM_NOSLEEP); 949 if (vq->vq_entries == NULL) { 950 r = ENOMEM; 951 goto err; 952 } 953 954 virtio_init_vq(sc, vq, false); 955 956 aprint_verbose_dev(sc->sc_dev, 957 "allocated %u byte for virtqueue %d for %s, " 958 "size %d\n", allocsize, index, name, vq_size); 959 if (allocsize3 > 0) 960 aprint_verbose_dev(sc->sc_dev, 961 "using %d byte (%d entries) " 962 "indirect descriptors\n", 963 allocsize3, maxnsegs * vq_size); 964 return 0; 965 966err: 967 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 968 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 969 if (vq->vq_dmamap) 970 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 971 if (vq->vq_vaddr) 972 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 973 if (vq->vq_segs[0].ds_addr) 974 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 975 memset(vq, 0, sizeof(*vq)); 976 977 return -1; 978} 979 980int 981virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 982{ 983 struct vq_entry *qe; 984 int i = 0; 985 986 /* device must be already deactivated */ 987 /* confirm the vq is empty */ 988 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 989 i++; 990 } 991 if (i != vq->vq_num) { 992 printf("%s: freeing non-empty vq, index %d\n", 993 device_xname(sc->sc_dev), vq->vq_index); 994 return EBUSY; 995 } 996 997 /* tell device that there's no virtqueue any longer */ 998 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 999 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 1000 nbo_bus_space_write_4(sc->sc_iot, sc->sc_ioh, 1001 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 1002 1003 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 1004 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 1005 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 1006 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 1007 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 1008 mutex_destroy(&vq->vq_freelist_lock); 1009 mutex_destroy(&vq->vq_uring_lock); 1010 mutex_destroy(&vq->vq_aring_lock); 1011 memset(vq, 0, sizeof(*vq)); 1012 1013 return 0; 1014} 1015 1016/* 1017 * Free descriptor management. 1018 */ 1019static struct vq_entry * 1020vq_alloc_entry(struct virtqueue *vq) 1021{ 1022 struct vq_entry *qe; 1023 1024 mutex_enter(&vq->vq_freelist_lock); 1025 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 1026 mutex_exit(&vq->vq_freelist_lock); 1027 return NULL; 1028 } 1029 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 1030 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 1031 mutex_exit(&vq->vq_freelist_lock); 1032 1033 return qe; 1034} 1035 1036static void 1037vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 1038{ 1039 mutex_enter(&vq->vq_freelist_lock); 1040 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 1041 mutex_exit(&vq->vq_freelist_lock); 1042 1043 return; 1044} 1045 1046/* 1047 * Enqueue several dmamaps as a single request. 1048 */ 1049/* 1050 * Typical usage: 1051 * <queue size> number of followings are stored in arrays 1052 * - command blocks (in dmamem) should be pre-allocated and mapped 1053 * - dmamaps for command blocks should be pre-allocated and loaded 1054 * - dmamaps for payload should be pre-allocated 1055 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1056 * if (r) // currently 0 or EAGAIN 1057 * return r; 1058 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1059 * if (r) { 1060 * virtio_enqueue_abort(sc, vq, slot); 1061 * return r; 1062 * } 1063 * r = virtio_enqueue_reserve(sc, vq, slot, 1064 * dmamap_payload[slot]->dm_nsegs+1); 1065 * // ^ +1 for command 1066 * if (r) { // currently 0 or EAGAIN 1067 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1068 * return r; // do not call abort() 1069 * } 1070 * <setup and prepare commands> 1071 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1072 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1073 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1074 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1075 * virtio_enqueue_commit(sc, vq, slot, true); 1076 */ 1077 1078/* 1079 * enqueue_prep: allocate a slot number 1080 */ 1081int 1082virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1083{ 1084 struct vq_entry *qe1; 1085 1086 KASSERT(slotp != NULL); 1087 1088 qe1 = vq_alloc_entry(vq); 1089 if (qe1 == NULL) 1090 return EAGAIN; 1091 /* next slot is not allocated yet */ 1092 qe1->qe_next = -1; 1093 *slotp = qe1->qe_index; 1094 1095 return 0; 1096} 1097 1098/* 1099 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1100 */ 1101int 1102virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1103 int slot, int nsegs) 1104{ 1105 int indirect; 1106 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1107 1108 KASSERT(qe1->qe_next == -1); 1109 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1110 1111 if ((vq->vq_indirect != NULL) && 1112 (nsegs >= MINSEG_INDIRECT) && 1113 (nsegs <= vq->vq_maxnsegs)) 1114 indirect = 1; 1115 else 1116 indirect = 0; 1117 qe1->qe_indirect = indirect; 1118 1119 if (indirect) { 1120 struct vring_desc *vd; 1121 int i; 1122 1123 vd = &vq->vq_desc[qe1->qe_index]; 1124 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1125 + vq->vq_indirectoffset; 1126 vd->addr += sizeof(struct vring_desc) 1127 * vq->vq_maxnsegs * qe1->qe_index; 1128 vd->len = sizeof(struct vring_desc) * nsegs; 1129 vd->flags = VRING_DESC_F_INDIRECT; 1130 1131 vd = vq->vq_indirect; 1132 vd += vq->vq_maxnsegs * qe1->qe_index; 1133 qe1->qe_desc_base = vd; 1134 1135 for (i = 0; i < nsegs-1; i++) { 1136 vd[i].flags = VRING_DESC_F_NEXT; 1137 } 1138 vd[i].flags = 0; 1139 qe1->qe_next = 0; 1140 1141 return 0; 1142 } else { 1143 struct vring_desc *vd; 1144 struct vq_entry *qe; 1145 int i, s; 1146 1147 vd = &vq->vq_desc[0]; 1148 qe1->qe_desc_base = vd; 1149 qe1->qe_next = qe1->qe_index; 1150 s = slot; 1151 for (i = 0; i < nsegs - 1; i++) { 1152 qe = vq_alloc_entry(vq); 1153 if (qe == NULL) { 1154 vd[s].flags = 0; 1155 virtio_enqueue_abort(sc, vq, slot); 1156 return EAGAIN; 1157 } 1158 vd[s].flags = VRING_DESC_F_NEXT; 1159 vd[s].next = qe->qe_index; 1160 s = qe->qe_index; 1161 } 1162 vd[s].flags = 0; 1163 1164 return 0; 1165 } 1166} 1167 1168/* 1169 * enqueue: enqueue a single dmamap. 1170 */ 1171int 1172virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1173 bus_dmamap_t dmamap, bool write) 1174{ 1175 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1176 struct vring_desc *vd = qe1->qe_desc_base; 1177 int i; 1178 int s = qe1->qe_next; 1179 1180 KASSERT(s >= 0); 1181 KASSERT(dmamap->dm_nsegs > 0); 1182 1183 for (i = 0; i < dmamap->dm_nsegs; i++) { 1184 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1185 vd[s].len = dmamap->dm_segs[i].ds_len; 1186 if (!write) 1187 vd[s].flags |= VRING_DESC_F_WRITE; 1188 s = vd[s].next; 1189 } 1190 qe1->qe_next = s; 1191 1192 return 0; 1193} 1194 1195int 1196virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1197 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1198 bool write) 1199{ 1200 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1201 struct vring_desc *vd = qe1->qe_desc_base; 1202 int s = qe1->qe_next; 1203 1204 KASSERT(s >= 0); 1205 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1206 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1207 (dmamap->dm_segs[0].ds_len >= start + len)); 1208 1209 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1210 vd[s].len = len; 1211 if (!write) 1212 vd[s].flags |= VRING_DESC_F_WRITE; 1213 qe1->qe_next = vd[s].next; 1214 1215 return 0; 1216} 1217 1218/* 1219 * enqueue_commit: add it to the aring. 1220 */ 1221int 1222virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1223 bool notifynow) 1224{ 1225 struct vq_entry *qe1; 1226 1227 if (slot < 0) { 1228 mutex_enter(&vq->vq_aring_lock); 1229 goto notify; 1230 } 1231 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1232 qe1 = &vq->vq_entries[slot]; 1233 if (qe1->qe_indirect) 1234 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1235 mutex_enter(&vq->vq_aring_lock); 1236 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1237 1238notify: 1239 if (notifynow) { 1240 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1241 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1242 membar_producer(); 1243 vq->vq_avail->idx = vq->vq_avail_idx; 1244 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1245 membar_producer(); 1246 vq->vq_queued++; 1247 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1248 membar_consumer(); 1249 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1250 nbo_bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1251 VIRTIO_CONFIG_QUEUE_NOTIFY, 1252 vq->vq_index); 1253 } 1254 mutex_exit(&vq->vq_aring_lock); 1255 1256 return 0; 1257} 1258 1259/* 1260 * enqueue_abort: rollback. 1261 */ 1262int 1263virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1264{ 1265 struct vq_entry *qe = &vq->vq_entries[slot]; 1266 struct vring_desc *vd; 1267 int s; 1268 1269 if (qe->qe_next < 0) { 1270 vq_free_entry(vq, qe); 1271 return 0; 1272 } 1273 1274 s = slot; 1275 vd = &vq->vq_desc[0]; 1276 while (vd[s].flags & VRING_DESC_F_NEXT) { 1277 s = vd[s].next; 1278 vq_free_entry(vq, qe); 1279 qe = &vq->vq_entries[s]; 1280 } 1281 vq_free_entry(vq, qe); 1282 return 0; 1283} 1284 1285/* 1286 * Dequeue a request. 1287 */ 1288/* 1289 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1290 * already done in the interrupt handler. 1291 */ 1292int 1293virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1294 int *slotp, int *lenp) 1295{ 1296 uint16_t slot, usedidx; 1297 struct vq_entry *qe; 1298 1299 if (vq->vq_used_idx == vq->vq_used->idx) 1300 return ENOENT; 1301 mutex_enter(&vq->vq_uring_lock); 1302 usedidx = vq->vq_used_idx++; 1303 mutex_exit(&vq->vq_uring_lock); 1304 usedidx %= vq->vq_num; 1305 slot = vq->vq_used->ring[usedidx].id; 1306 qe = &vq->vq_entries[slot]; 1307 1308 if (qe->qe_indirect) 1309 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1310 1311 if (slotp) 1312 *slotp = slot; 1313 if (lenp) 1314 *lenp = vq->vq_used->ring[usedidx].len; 1315 1316 return 0; 1317} 1318 1319/* 1320 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1321 * if you forget to call this the slot will be leaked. 1322 */ 1323int 1324virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1325{ 1326 struct vq_entry *qe = &vq->vq_entries[slot]; 1327 struct vring_desc *vd = &vq->vq_desc[0]; 1328 int s = slot; 1329 1330 while (vd[s].flags & VRING_DESC_F_NEXT) { 1331 s = vd[s].next; 1332 vq_free_entry(vq, qe); 1333 qe = &vq->vq_entries[s]; 1334 } 1335 vq_free_entry(vq, qe); 1336 1337 return 0; 1338} 1339 1340MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 1341 1342#ifdef _MODULE 1343#include "ioconf.c" 1344#endif 1345 1346static int 1347virtio_modcmd(modcmd_t cmd, void *opaque) 1348{ 1349 int error = 0; 1350 1351#ifdef _MODULE 1352 switch (cmd) { 1353 case MODULE_CMD_INIT: 1354 error = config_init_component(cfdriver_ioconf_virtio, 1355 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1356 break; 1357 case MODULE_CMD_FINI: 1358 error = config_fini_component(cfdriver_ioconf_virtio, 1359 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1360 break; 1361 default: 1362 error = ENOTTY; 1363 break; 1364 } 1365#endif 1366 1367 return error; 1368} 1369