1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* Driver for VirtIO memory balloon devices. */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD$"); 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/kernel.h> 37#include <sys/endian.h> 38#include <sys/kthread.h> 39#include <sys/malloc.h> 40#include <sys/module.h> 41#include <sys/sglist.h> 42#include <sys/sysctl.h> 43#include <sys/lock.h> 44#include <sys/mutex.h> 45#include <sys/queue.h> 46 47#include <vm/vm.h> 48#include <vm/vm_page.h> 49 50#include <machine/bus.h> 51#include <machine/resource.h> 52#include <sys/bus.h> 53#include <sys/rman.h> 54 55#include <dev/virtio/virtio.h> 56#include <dev/virtio/virtqueue.h> 57#include <dev/virtio/balloon/virtio_balloon.h> 58 59#include "virtio_if.h" 60 61struct vtballoon_softc { 62 device_t vtballoon_dev; 63 struct mtx vtballoon_mtx; 64 uint64_t vtballoon_features; 65 uint32_t vtballoon_flags; 66#define VTBALLOON_FLAG_DETACH 0x01 67 68 struct virtqueue *vtballoon_inflate_vq; 69 struct virtqueue *vtballoon_deflate_vq; 70 71 uint32_t vtballoon_desired_npages; 72 uint32_t vtballoon_current_npages; 73 TAILQ_HEAD(,vm_page) vtballoon_pages; 74 75 struct thread *vtballoon_td; 76 uint32_t *vtballoon_page_frames; 77 int vtballoon_timeout; 78}; 79 80static struct virtio_feature_desc vtballoon_feature_desc[] = { 81 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 82 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 83 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, 84 85 { 0, NULL } 86}; 87 88static int vtballoon_probe(device_t); 89static int vtballoon_attach(device_t); 90static int vtballoon_detach(device_t); 91static int vtballoon_config_change(device_t); 92 93static int vtballoon_negotiate_features(struct vtballoon_softc *); 94static int vtballoon_setup_features(struct vtballoon_softc *); 95static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 96 97static void vtballoon_vq_intr(void *); 98 99static void vtballoon_inflate(struct vtballoon_softc *, int); 100static void vtballoon_deflate(struct vtballoon_softc *, int); 101 102static void vtballoon_send_page_frames(struct vtballoon_softc *, 103 struct virtqueue *, int); 104 105static void vtballoon_pop(struct vtballoon_softc *); 106static void vtballoon_stop(struct vtballoon_softc *); 107 108static vm_page_t 109 vtballoon_alloc_page(struct vtballoon_softc *); 110static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 111 112static int vtballoon_sleep(struct vtballoon_softc *); 113static void vtballoon_thread(void *); 114static void vtballoon_setup_sysctl(struct vtballoon_softc *); 115 116#define vtballoon_modern(_sc) \ 117 (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) 118 119/* Features desired/implemented by this driver. */ 120#define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST 121 122/* Timeout between retries when the balloon needs inflating. */ 123#define VTBALLOON_LOWMEM_TIMEOUT hz 124 125/* 126 * Maximum number of pages we'll request to inflate or deflate 127 * the balloon in one virtqueue request. Both Linux and NetBSD 128 * have settled on 256, doing up to 1MB at a time. 129 */ 130#define VTBALLOON_PAGES_PER_REQUEST 256 131 132/* Must be able to fix all pages frames in one page (segment). */ 133CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 134 135#define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx 136#define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ 137 "VirtIO Balloon Lock", MTX_DEF) 138#define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) 139#define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) 140#define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) 141 142static device_method_t vtballoon_methods[] = { 143 /* Device methods. */ 144 DEVMETHOD(device_probe, vtballoon_probe), 145 DEVMETHOD(device_attach, vtballoon_attach), 146 DEVMETHOD(device_detach, vtballoon_detach), 147 148 /* VirtIO methods. */ 149 DEVMETHOD(virtio_config_change, vtballoon_config_change), 150 151 DEVMETHOD_END 152}; 153 154static driver_t vtballoon_driver = { 155 "vtballoon", 156 vtballoon_methods, 157 sizeof(struct vtballoon_softc) 158}; 159static devclass_t vtballoon_devclass; 160 161VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 162 vtballoon_devclass, 0, 0); 163MODULE_VERSION(virtio_balloon, 1); 164MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 165 166VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON, 167 "VirtIO Balloon Adapter"); 168 169static int 170vtballoon_probe(device_t dev) 171{ 172 return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon)); 173} 174 175static int 176vtballoon_attach(device_t dev) 177{ 178 struct vtballoon_softc *sc; 179 int error; 180 181 sc = device_get_softc(dev); 182 sc->vtballoon_dev = dev; 183 virtio_set_feature_desc(dev, vtballoon_feature_desc); 184 185 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); 186 TAILQ_INIT(&sc->vtballoon_pages); 187 188 vtballoon_setup_sysctl(sc); 189 190 error = vtballoon_setup_features(sc); 191 if (error) { 192 device_printf(dev, "cannot setup features\n"); 193 goto fail; 194 } 195 196 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * 197 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); 198 if (sc->vtballoon_page_frames == NULL) { 199 error = ENOMEM; 200 device_printf(dev, 201 "cannot allocate page frame request array\n"); 202 goto fail; 203 } 204 205 error = vtballoon_alloc_virtqueues(sc); 206 if (error) { 207 device_printf(dev, "cannot allocate virtqueues\n"); 208 goto fail; 209 } 210 211 error = virtio_setup_intr(dev, INTR_TYPE_MISC); 212 if (error) { 213 device_printf(dev, "cannot setup virtqueue interrupts\n"); 214 goto fail; 215 } 216 217 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 218 0, 0, "virtio_balloon"); 219 if (error) { 220 device_printf(dev, "cannot create balloon kthread\n"); 221 goto fail; 222 } 223 224 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 225 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 226 227fail: 228 if (error) 229 vtballoon_detach(dev); 230 231 return (error); 232} 233 234static int 235vtballoon_detach(device_t dev) 236{ 237 struct vtballoon_softc *sc; 238 239 sc = device_get_softc(dev); 240 241 if (sc->vtballoon_td != NULL) { 242 VTBALLOON_LOCK(sc); 243 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 244 wakeup_one(sc); 245 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); 246 VTBALLOON_UNLOCK(sc); 247 248 sc->vtballoon_td = NULL; 249 } 250 251 if (device_is_attached(dev)) { 252 vtballoon_pop(sc); 253 vtballoon_stop(sc); 254 } 255 256 if (sc->vtballoon_page_frames != NULL) { 257 free(sc->vtballoon_page_frames, M_DEVBUF); 258 sc->vtballoon_page_frames = NULL; 259 } 260 261 VTBALLOON_LOCK_DESTROY(sc); 262 263 return (0); 264} 265 266static int 267vtballoon_config_change(device_t dev) 268{ 269 struct vtballoon_softc *sc; 270 271 sc = device_get_softc(dev); 272 273 VTBALLOON_LOCK(sc); 274 wakeup_one(sc); 275 VTBALLOON_UNLOCK(sc); 276 277 return (1); 278} 279 280static int 281vtballoon_negotiate_features(struct vtballoon_softc *sc) 282{ 283 device_t dev; 284 uint64_t features; 285 286 dev = sc->vtballoon_dev; 287 features = VTBALLOON_FEATURES; 288 289 sc->vtballoon_features = virtio_negotiate_features(dev, features); 290 return (virtio_finalize_features(dev)); 291} 292 293static int 294vtballoon_setup_features(struct vtballoon_softc *sc) 295{ 296 int error; 297 298 error = vtballoon_negotiate_features(sc); 299 if (error) 300 return (error); 301 302 return (0); 303} 304 305static int 306vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 307{ 308 device_t dev; 309 struct vq_alloc_info vq_info[2]; 310 int nvqs; 311 312 dev = sc->vtballoon_dev; 313 nvqs = 2; 314 315 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, 316 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); 317 318 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, 319 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); 320 321 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 322} 323 324static void 325vtballoon_vq_intr(void *xsc) 326{ 327 struct vtballoon_softc *sc; 328 329 sc = xsc; 330 331 VTBALLOON_LOCK(sc); 332 wakeup_one(sc); 333 VTBALLOON_UNLOCK(sc); 334} 335 336static void 337vtballoon_inflate(struct vtballoon_softc *sc, int npages) 338{ 339 struct virtqueue *vq; 340 vm_page_t m; 341 int i; 342 343 vq = sc->vtballoon_inflate_vq; 344 345 if (npages > VTBALLOON_PAGES_PER_REQUEST) 346 npages = VTBALLOON_PAGES_PER_REQUEST; 347 348 for (i = 0; i < npages; i++) { 349 if ((m = vtballoon_alloc_page(sc)) == NULL) { 350 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 351 break; 352 } 353 354 sc->vtballoon_page_frames[i] = 355 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 356 357 KASSERT(m->a.queue == PQ_NONE, 358 ("%s: allocated page %p on queue", __func__, m)); 359 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); 360 } 361 362 if (i > 0) 363 vtballoon_send_page_frames(sc, vq, i); 364} 365 366static void 367vtballoon_deflate(struct vtballoon_softc *sc, int npages) 368{ 369 TAILQ_HEAD(, vm_page) free_pages; 370 struct virtqueue *vq; 371 vm_page_t m; 372 int i; 373 374 vq = sc->vtballoon_deflate_vq; 375 TAILQ_INIT(&free_pages); 376 377 if (npages > VTBALLOON_PAGES_PER_REQUEST) 378 npages = VTBALLOON_PAGES_PER_REQUEST; 379 380 for (i = 0; i < npages; i++) { 381 m = TAILQ_FIRST(&sc->vtballoon_pages); 382 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 383 384 sc->vtballoon_page_frames[i] = 385 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 386 387 TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); 388 TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); 389 } 390 391 if (i > 0) { 392 /* Always tell host first before freeing the pages. */ 393 vtballoon_send_page_frames(sc, vq, i); 394 395 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 396 TAILQ_REMOVE(&free_pages, m, plinks.q); 397 vtballoon_free_page(sc, m); 398 } 399 } 400 401 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 402 sc->vtballoon_current_npages == 0) || 403 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 404 sc->vtballoon_current_npages != 0), 405 ("%s: bogus page count %d", __func__, 406 sc->vtballoon_current_npages)); 407} 408 409static void 410vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 411 int npages) 412{ 413 struct sglist sg; 414 struct sglist_seg segs[1]; 415 void *c; 416 int error; 417 418 sglist_init(&sg, 1, segs); 419 420 error = sglist_append(&sg, sc->vtballoon_page_frames, 421 npages * sizeof(uint32_t)); 422 KASSERT(error == 0, ("error adding page frames to sglist")); 423 424 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 425 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 426 virtqueue_notify(vq); 427 428 /* 429 * Inflate and deflate operations are done synchronously. The 430 * interrupt handler will wake us up. 431 */ 432 VTBALLOON_LOCK(sc); 433 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) 434 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); 435 VTBALLOON_UNLOCK(sc); 436 437 KASSERT(c == vq, ("unexpected balloon operation response")); 438} 439 440static void 441vtballoon_pop(struct vtballoon_softc *sc) 442{ 443 444 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 445 vtballoon_deflate(sc, sc->vtballoon_current_npages); 446} 447 448static void 449vtballoon_stop(struct vtballoon_softc *sc) 450{ 451 452 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 453 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 454 455 virtio_stop(sc->vtballoon_dev); 456} 457 458static vm_page_t 459vtballoon_alloc_page(struct vtballoon_softc *sc) 460{ 461 vm_page_t m; 462 463 m = vm_page_alloc(NULL, 0, 464 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP); 465 if (m != NULL) 466 sc->vtballoon_current_npages++; 467 468 return (m); 469} 470 471static void 472vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 473{ 474 475 vm_page_free(m); 476 sc->vtballoon_current_npages--; 477} 478 479static uint32_t 480vtballoon_desired_size(struct vtballoon_softc *sc) 481{ 482 uint32_t desired; 483 484 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 485 offsetof(struct virtio_balloon_config, num_pages)); 486 487 if (vtballoon_modern(sc)) 488 return (desired); 489 else 490 return (le32toh(desired)); 491} 492 493static void 494vtballoon_update_size(struct vtballoon_softc *sc) 495{ 496 uint32_t npages; 497 498 npages = sc->vtballoon_current_npages; 499 if (!vtballoon_modern(sc)) 500 npages = htole32(npages); 501 502 virtio_write_dev_config_4(sc->vtballoon_dev, 503 offsetof(struct virtio_balloon_config, actual), npages); 504} 505 506static int 507vtballoon_sleep(struct vtballoon_softc *sc) 508{ 509 int rc, timeout; 510 uint32_t current, desired; 511 512 rc = 0; 513 current = sc->vtballoon_current_npages; 514 515 VTBALLOON_LOCK(sc); 516 for (;;) { 517 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 518 rc = 1; 519 break; 520 } 521 522 desired = vtballoon_desired_size(sc); 523 sc->vtballoon_desired_npages = desired; 524 525 /* 526 * If given, use non-zero timeout on the first time through 527 * the loop. On subsequent times, timeout will be zero so 528 * we will reevaluate the desired size of the balloon and 529 * break out to retry if needed. 530 */ 531 timeout = sc->vtballoon_timeout; 532 sc->vtballoon_timeout = 0; 533 534 if (current > desired) 535 break; 536 if (current < desired && timeout == 0) 537 break; 538 539 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); 540 } 541 VTBALLOON_UNLOCK(sc); 542 543 return (rc); 544} 545 546static void 547vtballoon_thread(void *xsc) 548{ 549 struct vtballoon_softc *sc; 550 uint32_t current, desired; 551 552 sc = xsc; 553 554 for (;;) { 555 if (vtballoon_sleep(sc) != 0) 556 break; 557 558 current = sc->vtballoon_current_npages; 559 desired = sc->vtballoon_desired_npages; 560 561 if (desired != current) { 562 if (desired > current) 563 vtballoon_inflate(sc, desired - current); 564 else 565 vtballoon_deflate(sc, current - desired); 566 567 vtballoon_update_size(sc); 568 } 569 } 570 571 kthread_exit(); 572} 573 574static void 575vtballoon_setup_sysctl(struct vtballoon_softc *sc) 576{ 577 device_t dev; 578 struct sysctl_ctx_list *ctx; 579 struct sysctl_oid *tree; 580 struct sysctl_oid_list *child; 581 582 dev = sc->vtballoon_dev; 583 ctx = device_get_sysctl_ctx(dev); 584 tree = device_get_sysctl_tree(dev); 585 child = SYSCTL_CHILDREN(tree); 586 587 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 588 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 589 "Desired balloon size in pages"); 590 591 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 592 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 593 "Current balloon size in pages"); 594} 595