31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/endian.h>
36#include <sys/kthread.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/sglist.h>
40#include <sys/sysctl.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/queue.h>
44
45#include <vm/vm.h>
46#include <vm/vm_page.h>
47
48#include <machine/bus.h>
49#include <machine/resource.h>
50#include <sys/bus.h>
51#include <sys/rman.h>
52
53#include <dev/virtio/virtio.h>
54#include <dev/virtio/virtqueue.h>
55#include <dev/virtio/balloon/virtio_balloon.h>
56
57#include "virtio_if.h"
58
59struct vtballoon_softc {
60 device_t vtballoon_dev;
61 struct mtx vtballoon_mtx;
62 uint64_t vtballoon_features;
63 uint32_t vtballoon_flags;
64#define VTBALLOON_FLAG_DETACH 0x01
65
66 struct virtqueue *vtballoon_inflate_vq;
67 struct virtqueue *vtballoon_deflate_vq;
68
69 uint32_t vtballoon_desired_npages;
70 uint32_t vtballoon_current_npages;
71 TAILQ_HEAD(,vm_page) vtballoon_pages;
72
73 struct thread *vtballoon_td;
74 uint32_t *vtballoon_page_frames;
75 int vtballoon_timeout;
76};
77
78static struct virtio_feature_desc vtballoon_feature_desc[] = {
79 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
80 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
81
82 { 0, NULL }
83};
84
85static int vtballoon_probe(device_t);
86static int vtballoon_attach(device_t);
87static int vtballoon_detach(device_t);
88static int vtballoon_config_change(device_t);
89
90static void vtballoon_negotiate_features(struct vtballoon_softc *);
91static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
92
93static void vtballoon_vq_intr(void *);
94
95static void vtballoon_inflate(struct vtballoon_softc *, int);
96static void vtballoon_deflate(struct vtballoon_softc *, int);
97
98static void vtballoon_send_page_frames(struct vtballoon_softc *,
99 struct virtqueue *, int);
100
101static void vtballoon_pop(struct vtballoon_softc *);
102static void vtballoon_stop(struct vtballoon_softc *);
103
104static vm_page_t
105 vtballoon_alloc_page(struct vtballoon_softc *);
106static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
107
108static int vtballoon_sleep(struct vtballoon_softc *);
109static void vtballoon_thread(void *);
110static void vtballoon_add_sysctl(struct vtballoon_softc *);
111
112/* Features desired/implemented by this driver. */
113#define VTBALLOON_FEATURES 0
114
115/* Timeout between retries when the balloon needs inflating. */
116#define VTBALLOON_LOWMEM_TIMEOUT hz
117
118/*
119 * Maximum number of pages we'll request to inflate or deflate
120 * the balloon in one virtqueue request. Both Linux and NetBSD
121 * have settled on 256, doing up to 1MB at a time.
122 */
123#define VTBALLOON_PAGES_PER_REQUEST 256
124
125/* Must be able to fix all pages frames in one page (segment). */
126CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
127
128#define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
129#define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
130 "VirtIO Balloon Lock", MTX_DEF)
131#define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
132#define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
133#define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
134
135static device_method_t vtballoon_methods[] = {
136 /* Device methods. */
137 DEVMETHOD(device_probe, vtballoon_probe),
138 DEVMETHOD(device_attach, vtballoon_attach),
139 DEVMETHOD(device_detach, vtballoon_detach),
140
141 /* VirtIO methods. */
142 DEVMETHOD(virtio_config_change, vtballoon_config_change),
143
144 DEVMETHOD_END
145};
146
147static driver_t vtballoon_driver = {
148 "vtballoon",
149 vtballoon_methods,
150 sizeof(struct vtballoon_softc)
151};
152static devclass_t vtballoon_devclass;
153
154DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
155 vtballoon_devclass, 0, 0);
156MODULE_VERSION(virtio_balloon, 1);
157MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
158
159static int
160vtballoon_probe(device_t dev)
161{
162
163 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
164 return (ENXIO);
165
166 device_set_desc(dev, "VirtIO Balloon Adapter");
167
168 return (BUS_PROBE_DEFAULT);
169}
170
171static int
172vtballoon_attach(device_t dev)
173{
174 struct vtballoon_softc *sc;
175 int error;
176
177 sc = device_get_softc(dev);
178 sc->vtballoon_dev = dev;
179
180 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
181 TAILQ_INIT(&sc->vtballoon_pages);
182
183 vtballoon_add_sysctl(sc);
184
185 virtio_set_feature_desc(dev, vtballoon_feature_desc);
186 vtballoon_negotiate_features(sc);
187
188 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
189 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
190 if (sc->vtballoon_page_frames == NULL) {
191 error = ENOMEM;
192 device_printf(dev,
193 "cannot allocate page frame request array\n");
194 goto fail;
195 }
196
197 error = vtballoon_alloc_virtqueues(sc);
198 if (error) {
199 device_printf(dev, "cannot allocate virtqueues\n");
200 goto fail;
201 }
202
203 error = virtio_setup_intr(dev, INTR_TYPE_MISC);
204 if (error) {
205 device_printf(dev, "cannot setup virtqueue interrupts\n");
206 goto fail;
207 }
208
209 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
210 0, 0, "virtio_balloon");
211 if (error) {
212 device_printf(dev, "cannot create balloon kthread\n");
213 goto fail;
214 }
215
216 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
217 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
218
219fail:
220 if (error)
221 vtballoon_detach(dev);
222
223 return (error);
224}
225
226static int
227vtballoon_detach(device_t dev)
228{
229 struct vtballoon_softc *sc;
230
231 sc = device_get_softc(dev);
232
233 if (sc->vtballoon_td != NULL) {
234 VTBALLOON_LOCK(sc);
235 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
236 wakeup_one(sc);
237 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
238 VTBALLOON_UNLOCK(sc);
239
240 sc->vtballoon_td = NULL;
241 }
242
243 if (device_is_attached(dev)) {
244 vtballoon_pop(sc);
245 vtballoon_stop(sc);
246 }
247
248 if (sc->vtballoon_page_frames != NULL) {
249 free(sc->vtballoon_page_frames, M_DEVBUF);
250 sc->vtballoon_page_frames = NULL;
251 }
252
253 VTBALLOON_LOCK_DESTROY(sc);
254
255 return (0);
256}
257
258static int
259vtballoon_config_change(device_t dev)
260{
261 struct vtballoon_softc *sc;
262
263 sc = device_get_softc(dev);
264
265 VTBALLOON_LOCK(sc);
266 wakeup_one(sc);
267 VTBALLOON_UNLOCK(sc);
268
269 return (1);
270}
271
272static void
273vtballoon_negotiate_features(struct vtballoon_softc *sc)
274{
275 device_t dev;
276 uint64_t features;
277
278 dev = sc->vtballoon_dev;
279 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
280 sc->vtballoon_features = features;
281}
282
283static int
284vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
285{
286 device_t dev;
287 struct vq_alloc_info vq_info[2];
288 int nvqs;
289
290 dev = sc->vtballoon_dev;
291 nvqs = 2;
292
293 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
294 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
295
296 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
297 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
298
299 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
300}
301
302static void
303vtballoon_vq_intr(void *xsc)
304{
305 struct vtballoon_softc *sc;
306
307 sc = xsc;
308
309 VTBALLOON_LOCK(sc);
310 wakeup_one(sc);
311 VTBALLOON_UNLOCK(sc);
312}
313
314static void
315vtballoon_inflate(struct vtballoon_softc *sc, int npages)
316{
317 struct virtqueue *vq;
318 vm_page_t m;
319 int i;
320
321 vq = sc->vtballoon_inflate_vq;
322
323 if (npages > VTBALLOON_PAGES_PER_REQUEST)
324 npages = VTBALLOON_PAGES_PER_REQUEST;
325
326 for (i = 0; i < npages; i++) {
327 if ((m = vtballoon_alloc_page(sc)) == NULL) {
328 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
329 break;
330 }
331
332 sc->vtballoon_page_frames[i] =
333 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
334
335 KASSERT(m->queue == PQ_NONE,
336 ("%s: allocated page %p on queue", __func__, m));
337 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
338 }
339
340 if (i > 0)
341 vtballoon_send_page_frames(sc, vq, i);
342}
343
344static void
345vtballoon_deflate(struct vtballoon_softc *sc, int npages)
346{
347 TAILQ_HEAD(, vm_page) free_pages;
348 struct virtqueue *vq;
349 vm_page_t m;
350 int i;
351
352 vq = sc->vtballoon_deflate_vq;
353 TAILQ_INIT(&free_pages);
354
355 if (npages > VTBALLOON_PAGES_PER_REQUEST)
356 npages = VTBALLOON_PAGES_PER_REQUEST;
357
358 for (i = 0; i < npages; i++) {
359 m = TAILQ_FIRST(&sc->vtballoon_pages);
360 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
361
362 sc->vtballoon_page_frames[i] =
363 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
364
365 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
366 TAILQ_INSERT_TAIL(&free_pages, m, pageq);
367 }
368
369 if (i > 0) {
370 /* Always tell host first before freeing the pages. */
371 vtballoon_send_page_frames(sc, vq, i);
372
373 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
374 TAILQ_REMOVE(&free_pages, m, pageq);
375 vtballoon_free_page(sc, m);
376 }
377 }
378
379 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
380 sc->vtballoon_current_npages == 0) ||
381 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
382 sc->vtballoon_current_npages != 0),
383 ("%s: bogus page count %d", __func__,
384 sc->vtballoon_current_npages));
385}
386
387static void
388vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
389 int npages)
390{
391 struct sglist sg;
392 struct sglist_seg segs[1];
393 void *c;
394 int error;
395
396 sglist_init(&sg, 1, segs);
397
398 error = sglist_append(&sg, sc->vtballoon_page_frames,
399 npages * sizeof(uint32_t));
400 KASSERT(error == 0, ("error adding page frames to sglist"));
401
402 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
403 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
404 virtqueue_notify(vq);
405
406 /*
407 * Inflate and deflate operations are done synchronously. The
408 * interrupt handler will wake us up.
409 */
410 VTBALLOON_LOCK(sc);
411 while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
412 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
413 VTBALLOON_UNLOCK(sc);
414
415 KASSERT(c == vq, ("unexpected balloon operation response"));
416}
417
418static void
419vtballoon_pop(struct vtballoon_softc *sc)
420{
421
422 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
423 vtballoon_deflate(sc, sc->vtballoon_current_npages);
424}
425
426static void
427vtballoon_stop(struct vtballoon_softc *sc)
428{
429
430 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
431 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
432
433 virtio_stop(sc->vtballoon_dev);
434}
435
436static vm_page_t
437vtballoon_alloc_page(struct vtballoon_softc *sc)
438{
439 vm_page_t m;
440
441 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
442 VM_ALLOC_NOOBJ);
443 if (m != NULL)
444 sc->vtballoon_current_npages++;
445
446 return (m);
447}
448
449static void
450vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
451{
452
453 vm_page_unwire(m, 0);
454 vm_page_free(m);
455 sc->vtballoon_current_npages--;
456}
457
458static uint32_t
459vtballoon_desired_size(struct vtballoon_softc *sc)
460{
461 uint32_t desired;
462
463 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
464 offsetof(struct virtio_balloon_config, num_pages));
465
466 return (le32toh(desired));
467}
468
469static void
470vtballoon_update_size(struct vtballoon_softc *sc)
471{
472
473 virtio_write_dev_config_4(sc->vtballoon_dev,
474 offsetof(struct virtio_balloon_config, actual),
475 htole32(sc->vtballoon_current_npages));
476}
477
478static int
479vtballoon_sleep(struct vtballoon_softc *sc)
480{
481 int rc, timeout;
482 uint32_t current, desired;
483
484 rc = 0;
485 current = sc->vtballoon_current_npages;
486
487 VTBALLOON_LOCK(sc);
488 for (;;) {
489 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
490 rc = 1;
491 break;
492 }
493
494 desired = vtballoon_desired_size(sc);
495 sc->vtballoon_desired_npages = desired;
496
497 /*
498 * If given, use non-zero timeout on the first time through
499 * the loop. On subsequent times, timeout will be zero so
500 * we will reevaluate the desired size of the balloon and
501 * break out to retry if needed.
502 */
503 timeout = sc->vtballoon_timeout;
504 sc->vtballoon_timeout = 0;
505
506 if (current > desired)
507 break;
508 if (current < desired && timeout == 0)
509 break;
510
511 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
512 }
513 VTBALLOON_UNLOCK(sc);
514
515 return (rc);
516}
517
518static void
519vtballoon_thread(void *xsc)
520{
521 struct vtballoon_softc *sc;
522 uint32_t current, desired;
523
524 sc = xsc;
525
526 for (;;) {
527 if (vtballoon_sleep(sc) != 0)
528 break;
529
530 current = sc->vtballoon_current_npages;
531 desired = sc->vtballoon_desired_npages;
532
533 if (desired != current) {
534 if (desired > current)
535 vtballoon_inflate(sc, desired - current);
536 else
537 vtballoon_deflate(sc, current - desired);
538
539 vtballoon_update_size(sc);
540 }
541 }
542
543 kthread_exit();
544}
545
546static void
547vtballoon_add_sysctl(struct vtballoon_softc *sc)
548{
549 device_t dev;
550 struct sysctl_ctx_list *ctx;
551 struct sysctl_oid *tree;
552 struct sysctl_oid_list *child;
553
554 dev = sc->vtballoon_dev;
555 ctx = device_get_sysctl_ctx(dev);
556 tree = device_get_sysctl_tree(dev);
557 child = SYSCTL_CHILDREN(tree);
558
559 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
560 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
561 "Desired balloon size in pages");
562
563 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
564 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
565 "Current balloon size in pages");
566}
| 31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/endian.h>
36#include <sys/kthread.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/sglist.h>
40#include <sys/sysctl.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/queue.h>
44
45#include <vm/vm.h>
46#include <vm/vm_page.h>
47
48#include <machine/bus.h>
49#include <machine/resource.h>
50#include <sys/bus.h>
51#include <sys/rman.h>
52
53#include <dev/virtio/virtio.h>
54#include <dev/virtio/virtqueue.h>
55#include <dev/virtio/balloon/virtio_balloon.h>
56
57#include "virtio_if.h"
58
59struct vtballoon_softc {
60 device_t vtballoon_dev;
61 struct mtx vtballoon_mtx;
62 uint64_t vtballoon_features;
63 uint32_t vtballoon_flags;
64#define VTBALLOON_FLAG_DETACH 0x01
65
66 struct virtqueue *vtballoon_inflate_vq;
67 struct virtqueue *vtballoon_deflate_vq;
68
69 uint32_t vtballoon_desired_npages;
70 uint32_t vtballoon_current_npages;
71 TAILQ_HEAD(,vm_page) vtballoon_pages;
72
73 struct thread *vtballoon_td;
74 uint32_t *vtballoon_page_frames;
75 int vtballoon_timeout;
76};
77
78static struct virtio_feature_desc vtballoon_feature_desc[] = {
79 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
80 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
81
82 { 0, NULL }
83};
84
85static int vtballoon_probe(device_t);
86static int vtballoon_attach(device_t);
87static int vtballoon_detach(device_t);
88static int vtballoon_config_change(device_t);
89
90static void vtballoon_negotiate_features(struct vtballoon_softc *);
91static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
92
93static void vtballoon_vq_intr(void *);
94
95static void vtballoon_inflate(struct vtballoon_softc *, int);
96static void vtballoon_deflate(struct vtballoon_softc *, int);
97
98static void vtballoon_send_page_frames(struct vtballoon_softc *,
99 struct virtqueue *, int);
100
101static void vtballoon_pop(struct vtballoon_softc *);
102static void vtballoon_stop(struct vtballoon_softc *);
103
104static vm_page_t
105 vtballoon_alloc_page(struct vtballoon_softc *);
106static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
107
108static int vtballoon_sleep(struct vtballoon_softc *);
109static void vtballoon_thread(void *);
110static void vtballoon_add_sysctl(struct vtballoon_softc *);
111
112/* Features desired/implemented by this driver. */
113#define VTBALLOON_FEATURES 0
114
115/* Timeout between retries when the balloon needs inflating. */
116#define VTBALLOON_LOWMEM_TIMEOUT hz
117
118/*
119 * Maximum number of pages we'll request to inflate or deflate
120 * the balloon in one virtqueue request. Both Linux and NetBSD
121 * have settled on 256, doing up to 1MB at a time.
122 */
123#define VTBALLOON_PAGES_PER_REQUEST 256
124
125/* Must be able to fix all pages frames in one page (segment). */
126CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
127
128#define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx
129#define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \
130 "VirtIO Balloon Lock", MTX_DEF)
131#define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc)))
132#define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc)))
133#define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc)))
134
135static device_method_t vtballoon_methods[] = {
136 /* Device methods. */
137 DEVMETHOD(device_probe, vtballoon_probe),
138 DEVMETHOD(device_attach, vtballoon_attach),
139 DEVMETHOD(device_detach, vtballoon_detach),
140
141 /* VirtIO methods. */
142 DEVMETHOD(virtio_config_change, vtballoon_config_change),
143
144 DEVMETHOD_END
145};
146
147static driver_t vtballoon_driver = {
148 "vtballoon",
149 vtballoon_methods,
150 sizeof(struct vtballoon_softc)
151};
152static devclass_t vtballoon_devclass;
153
154DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
155 vtballoon_devclass, 0, 0);
156MODULE_VERSION(virtio_balloon, 1);
157MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
158
159static int
160vtballoon_probe(device_t dev)
161{
162
163 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
164 return (ENXIO);
165
166 device_set_desc(dev, "VirtIO Balloon Adapter");
167
168 return (BUS_PROBE_DEFAULT);
169}
170
171static int
172vtballoon_attach(device_t dev)
173{
174 struct vtballoon_softc *sc;
175 int error;
176
177 sc = device_get_softc(dev);
178 sc->vtballoon_dev = dev;
179
180 VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
181 TAILQ_INIT(&sc->vtballoon_pages);
182
183 vtballoon_add_sysctl(sc);
184
185 virtio_set_feature_desc(dev, vtballoon_feature_desc);
186 vtballoon_negotiate_features(sc);
187
188 sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
189 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
190 if (sc->vtballoon_page_frames == NULL) {
191 error = ENOMEM;
192 device_printf(dev,
193 "cannot allocate page frame request array\n");
194 goto fail;
195 }
196
197 error = vtballoon_alloc_virtqueues(sc);
198 if (error) {
199 device_printf(dev, "cannot allocate virtqueues\n");
200 goto fail;
201 }
202
203 error = virtio_setup_intr(dev, INTR_TYPE_MISC);
204 if (error) {
205 device_printf(dev, "cannot setup virtqueue interrupts\n");
206 goto fail;
207 }
208
209 error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
210 0, 0, "virtio_balloon");
211 if (error) {
212 device_printf(dev, "cannot create balloon kthread\n");
213 goto fail;
214 }
215
216 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
217 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
218
219fail:
220 if (error)
221 vtballoon_detach(dev);
222
223 return (error);
224}
225
226static int
227vtballoon_detach(device_t dev)
228{
229 struct vtballoon_softc *sc;
230
231 sc = device_get_softc(dev);
232
233 if (sc->vtballoon_td != NULL) {
234 VTBALLOON_LOCK(sc);
235 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
236 wakeup_one(sc);
237 msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
238 VTBALLOON_UNLOCK(sc);
239
240 sc->vtballoon_td = NULL;
241 }
242
243 if (device_is_attached(dev)) {
244 vtballoon_pop(sc);
245 vtballoon_stop(sc);
246 }
247
248 if (sc->vtballoon_page_frames != NULL) {
249 free(sc->vtballoon_page_frames, M_DEVBUF);
250 sc->vtballoon_page_frames = NULL;
251 }
252
253 VTBALLOON_LOCK_DESTROY(sc);
254
255 return (0);
256}
257
258static int
259vtballoon_config_change(device_t dev)
260{
261 struct vtballoon_softc *sc;
262
263 sc = device_get_softc(dev);
264
265 VTBALLOON_LOCK(sc);
266 wakeup_one(sc);
267 VTBALLOON_UNLOCK(sc);
268
269 return (1);
270}
271
272static void
273vtballoon_negotiate_features(struct vtballoon_softc *sc)
274{
275 device_t dev;
276 uint64_t features;
277
278 dev = sc->vtballoon_dev;
279 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
280 sc->vtballoon_features = features;
281}
282
283static int
284vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
285{
286 device_t dev;
287 struct vq_alloc_info vq_info[2];
288 int nvqs;
289
290 dev = sc->vtballoon_dev;
291 nvqs = 2;
292
293 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
294 &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
295
296 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
297 &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
298
299 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
300}
301
302static void
303vtballoon_vq_intr(void *xsc)
304{
305 struct vtballoon_softc *sc;
306
307 sc = xsc;
308
309 VTBALLOON_LOCK(sc);
310 wakeup_one(sc);
311 VTBALLOON_UNLOCK(sc);
312}
313
314static void
315vtballoon_inflate(struct vtballoon_softc *sc, int npages)
316{
317 struct virtqueue *vq;
318 vm_page_t m;
319 int i;
320
321 vq = sc->vtballoon_inflate_vq;
322
323 if (npages > VTBALLOON_PAGES_PER_REQUEST)
324 npages = VTBALLOON_PAGES_PER_REQUEST;
325
326 for (i = 0; i < npages; i++) {
327 if ((m = vtballoon_alloc_page(sc)) == NULL) {
328 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
329 break;
330 }
331
332 sc->vtballoon_page_frames[i] =
333 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
334
335 KASSERT(m->queue == PQ_NONE,
336 ("%s: allocated page %p on queue", __func__, m));
337 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
338 }
339
340 if (i > 0)
341 vtballoon_send_page_frames(sc, vq, i);
342}
343
344static void
345vtballoon_deflate(struct vtballoon_softc *sc, int npages)
346{
347 TAILQ_HEAD(, vm_page) free_pages;
348 struct virtqueue *vq;
349 vm_page_t m;
350 int i;
351
352 vq = sc->vtballoon_deflate_vq;
353 TAILQ_INIT(&free_pages);
354
355 if (npages > VTBALLOON_PAGES_PER_REQUEST)
356 npages = VTBALLOON_PAGES_PER_REQUEST;
357
358 for (i = 0; i < npages; i++) {
359 m = TAILQ_FIRST(&sc->vtballoon_pages);
360 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
361
362 sc->vtballoon_page_frames[i] =
363 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
364
365 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
366 TAILQ_INSERT_TAIL(&free_pages, m, pageq);
367 }
368
369 if (i > 0) {
370 /* Always tell host first before freeing the pages. */
371 vtballoon_send_page_frames(sc, vq, i);
372
373 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
374 TAILQ_REMOVE(&free_pages, m, pageq);
375 vtballoon_free_page(sc, m);
376 }
377 }
378
379 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
380 sc->vtballoon_current_npages == 0) ||
381 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
382 sc->vtballoon_current_npages != 0),
383 ("%s: bogus page count %d", __func__,
384 sc->vtballoon_current_npages));
385}
386
387static void
388vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
389 int npages)
390{
391 struct sglist sg;
392 struct sglist_seg segs[1];
393 void *c;
394 int error;
395
396 sglist_init(&sg, 1, segs);
397
398 error = sglist_append(&sg, sc->vtballoon_page_frames,
399 npages * sizeof(uint32_t));
400 KASSERT(error == 0, ("error adding page frames to sglist"));
401
402 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
403 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
404 virtqueue_notify(vq);
405
406 /*
407 * Inflate and deflate operations are done synchronously. The
408 * interrupt handler will wake us up.
409 */
410 VTBALLOON_LOCK(sc);
411 while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
412 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
413 VTBALLOON_UNLOCK(sc);
414
415 KASSERT(c == vq, ("unexpected balloon operation response"));
416}
417
418static void
419vtballoon_pop(struct vtballoon_softc *sc)
420{
421
422 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
423 vtballoon_deflate(sc, sc->vtballoon_current_npages);
424}
425
426static void
427vtballoon_stop(struct vtballoon_softc *sc)
428{
429
430 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
431 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
432
433 virtio_stop(sc->vtballoon_dev);
434}
435
436static vm_page_t
437vtballoon_alloc_page(struct vtballoon_softc *sc)
438{
439 vm_page_t m;
440
441 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
442 VM_ALLOC_NOOBJ);
443 if (m != NULL)
444 sc->vtballoon_current_npages++;
445
446 return (m);
447}
448
449static void
450vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
451{
452
453 vm_page_unwire(m, 0);
454 vm_page_free(m);
455 sc->vtballoon_current_npages--;
456}
457
458static uint32_t
459vtballoon_desired_size(struct vtballoon_softc *sc)
460{
461 uint32_t desired;
462
463 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
464 offsetof(struct virtio_balloon_config, num_pages));
465
466 return (le32toh(desired));
467}
468
469static void
470vtballoon_update_size(struct vtballoon_softc *sc)
471{
472
473 virtio_write_dev_config_4(sc->vtballoon_dev,
474 offsetof(struct virtio_balloon_config, actual),
475 htole32(sc->vtballoon_current_npages));
476}
477
478static int
479vtballoon_sleep(struct vtballoon_softc *sc)
480{
481 int rc, timeout;
482 uint32_t current, desired;
483
484 rc = 0;
485 current = sc->vtballoon_current_npages;
486
487 VTBALLOON_LOCK(sc);
488 for (;;) {
489 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
490 rc = 1;
491 break;
492 }
493
494 desired = vtballoon_desired_size(sc);
495 sc->vtballoon_desired_npages = desired;
496
497 /*
498 * If given, use non-zero timeout on the first time through
499 * the loop. On subsequent times, timeout will be zero so
500 * we will reevaluate the desired size of the balloon and
501 * break out to retry if needed.
502 */
503 timeout = sc->vtballoon_timeout;
504 sc->vtballoon_timeout = 0;
505
506 if (current > desired)
507 break;
508 if (current < desired && timeout == 0)
509 break;
510
511 msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
512 }
513 VTBALLOON_UNLOCK(sc);
514
515 return (rc);
516}
517
518static void
519vtballoon_thread(void *xsc)
520{
521 struct vtballoon_softc *sc;
522 uint32_t current, desired;
523
524 sc = xsc;
525
526 for (;;) {
527 if (vtballoon_sleep(sc) != 0)
528 break;
529
530 current = sc->vtballoon_current_npages;
531 desired = sc->vtballoon_desired_npages;
532
533 if (desired != current) {
534 if (desired > current)
535 vtballoon_inflate(sc, desired - current);
536 else
537 vtballoon_deflate(sc, current - desired);
538
539 vtballoon_update_size(sc);
540 }
541 }
542
543 kthread_exit();
544}
545
546static void
547vtballoon_add_sysctl(struct vtballoon_softc *sc)
548{
549 device_t dev;
550 struct sysctl_ctx_list *ctx;
551 struct sysctl_oid *tree;
552 struct sysctl_oid_list *child;
553
554 dev = sc->vtballoon_dev;
555 ctx = device_get_sysctl_ctx(dev);
556 tree = device_get_sysctl_tree(dev);
557 child = SYSCTL_CHILDREN(tree);
558
559 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
560 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
561 "Desired balloon size in pages");
562
563 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
564 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
565 "Current balloon size in pages");
566}
|