virtio_balloon.c revision 331722 
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice unmodified, this list of conditions, and the following
10 *    disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27/* Driver for VirtIO memory balloon devices. */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/11/sys/dev/virtio/balloon/virtio_balloon.c 331722 2018-03-29 02:50:57Z eadler $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/endian.h>
36#include <sys/kthread.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/sglist.h>
40#include <sys/sysctl.h>
41#include <sys/lock.h>
42#include <sys/mutex.h>
43#include <sys/queue.h>
44
45#include <vm/vm.h>
46#include <vm/vm_page.h>
47
48#include <machine/bus.h>
49#include <machine/resource.h>
50#include <sys/bus.h>
51#include <sys/rman.h>
52
53#include <dev/virtio/virtio.h>
54#include <dev/virtio/virtqueue.h>
55#include <dev/virtio/balloon/virtio_balloon.h>
56
57#include "virtio_if.h"
58
59struct vtballoon_softc {
60	device_t		 vtballoon_dev;
61	struct mtx		 vtballoon_mtx;
62	uint64_t		 vtballoon_features;
63	uint32_t		 vtballoon_flags;
64#define VTBALLOON_FLAG_DETACH	 0x01
65
66	struct virtqueue	*vtballoon_inflate_vq;
67	struct virtqueue	*vtballoon_deflate_vq;
68
69	uint32_t		 vtballoon_desired_npages;
70	uint32_t		 vtballoon_current_npages;
71	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
72
73	struct thread		*vtballoon_td;
74	uint32_t		*vtballoon_page_frames;
75	int			 vtballoon_timeout;
76};
77
78static struct virtio_feature_desc vtballoon_feature_desc[] = {
79	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
80	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
81
82	{ 0, NULL }
83};
84
85static int	vtballoon_probe(device_t);
86static int	vtballoon_attach(device_t);
87static int	vtballoon_detach(device_t);
88static int	vtballoon_config_change(device_t);
89
90static void	vtballoon_negotiate_features(struct vtballoon_softc *);
91static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
92
93static void	vtballoon_vq_intr(void *);
94
95static void	vtballoon_inflate(struct vtballoon_softc *, int);
96static void	vtballoon_deflate(struct vtballoon_softc *, int);
97
98static void	vtballoon_send_page_frames(struct vtballoon_softc *,
99		    struct virtqueue *, int);
100
101static void	vtballoon_pop(struct vtballoon_softc *);
102static void	vtballoon_stop(struct vtballoon_softc *);
103
104static vm_page_t
105		vtballoon_alloc_page(struct vtballoon_softc *);
106static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
107
108static int	vtballoon_sleep(struct vtballoon_softc *);
109static void	vtballoon_thread(void *);
110static void	vtballoon_add_sysctl(struct vtballoon_softc *);
111
112/* Features desired/implemented by this driver. */
113#define VTBALLOON_FEATURES		0
114
115/* Timeout between retries when the balloon needs inflating. */
116#define VTBALLOON_LOWMEM_TIMEOUT	hz
117
118/*
119 * Maximum number of pages we'll request to inflate or deflate
120 * the balloon in one virtqueue request. Both Linux and NetBSD
121 * have settled on 256, doing up to 1MB at a time.
122 */
123#define VTBALLOON_PAGES_PER_REQUEST	256
124
125/* Must be able to fix all pages frames in one page (segment). */
126CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
127
128#define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
129#define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
130					    "VirtIO Balloon Lock", MTX_DEF)
131#define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
132#define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
133#define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
134
135static device_method_t vtballoon_methods[] = {
136	/* Device methods. */
137	DEVMETHOD(device_probe,		vtballoon_probe),
138	DEVMETHOD(device_attach,	vtballoon_attach),
139	DEVMETHOD(device_detach,	vtballoon_detach),
140
141	/* VirtIO methods. */
142	DEVMETHOD(virtio_config_change, vtballoon_config_change),
143
144	DEVMETHOD_END
145};
146
147static driver_t vtballoon_driver = {
148	"vtballoon",
149	vtballoon_methods,
150	sizeof(struct vtballoon_softc)
151};
152static devclass_t vtballoon_devclass;
153
154DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
155    vtballoon_devclass, 0, 0);
156MODULE_VERSION(virtio_balloon, 1);
157MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
158
159static int
160vtballoon_probe(device_t dev)
161{
162
163	if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
164		return (ENXIO);
165
166	device_set_desc(dev, "VirtIO Balloon Adapter");
167
168	return (BUS_PROBE_DEFAULT);
169}
170
171static int
172vtballoon_attach(device_t dev)
173{
174	struct vtballoon_softc *sc;
175	int error;
176
177	sc = device_get_softc(dev);
178	sc->vtballoon_dev = dev;
179
180	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
181	TAILQ_INIT(&sc->vtballoon_pages);
182
183	vtballoon_add_sysctl(sc);
184
185	virtio_set_feature_desc(dev, vtballoon_feature_desc);
186	vtballoon_negotiate_features(sc);
187
188	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
189	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
190	if (sc->vtballoon_page_frames == NULL) {
191		error = ENOMEM;
192		device_printf(dev,
193		    "cannot allocate page frame request array\n");
194		goto fail;
195	}
196
197	error = vtballoon_alloc_virtqueues(sc);
198	if (error) {
199		device_printf(dev, "cannot allocate virtqueues\n");
200		goto fail;
201	}
202
203	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
204	if (error) {
205		device_printf(dev, "cannot setup virtqueue interrupts\n");
206		goto fail;
207	}
208
209	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
210	    0, 0, "virtio_balloon");
211	if (error) {
212		device_printf(dev, "cannot create balloon kthread\n");
213		goto fail;
214	}
215
216	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
217	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
218
219fail:
220	if (error)
221		vtballoon_detach(dev);
222
223	return (error);
224}
225
226static int
227vtballoon_detach(device_t dev)
228{
229	struct vtballoon_softc *sc;
230
231	sc = device_get_softc(dev);
232
233	if (sc->vtballoon_td != NULL) {
234		VTBALLOON_LOCK(sc);
235		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
236		wakeup_one(sc);
237		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
238		VTBALLOON_UNLOCK(sc);
239
240		sc->vtballoon_td = NULL;
241	}
242
243	if (device_is_attached(dev)) {
244		vtballoon_pop(sc);
245		vtballoon_stop(sc);
246	}
247
248	if (sc->vtballoon_page_frames != NULL) {
249		free(sc->vtballoon_page_frames, M_DEVBUF);
250		sc->vtballoon_page_frames = NULL;
251	}
252
253	VTBALLOON_LOCK_DESTROY(sc);
254
255	return (0);
256}
257
258static int
259vtballoon_config_change(device_t dev)
260{
261	struct vtballoon_softc *sc;
262
263	sc = device_get_softc(dev);
264
265	VTBALLOON_LOCK(sc);
266	wakeup_one(sc);
267	VTBALLOON_UNLOCK(sc);
268
269	return (1);
270}
271
272static void
273vtballoon_negotiate_features(struct vtballoon_softc *sc)
274{
275	device_t dev;
276	uint64_t features;
277
278	dev = sc->vtballoon_dev;
279	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
280	sc->vtballoon_features = features;
281}
282
283static int
284vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
285{
286	device_t dev;
287	struct vq_alloc_info vq_info[2];
288	int nvqs;
289
290	dev = sc->vtballoon_dev;
291	nvqs = 2;
292
293	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
294	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
295
296	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
297	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
298
299	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
300}
301
302static void
303vtballoon_vq_intr(void *xsc)
304{
305	struct vtballoon_softc *sc;
306
307	sc = xsc;
308
309	VTBALLOON_LOCK(sc);
310	wakeup_one(sc);
311	VTBALLOON_UNLOCK(sc);
312}
313
314static void
315vtballoon_inflate(struct vtballoon_softc *sc, int npages)
316{
317	struct virtqueue *vq;
318	vm_page_t m;
319	int i;
320
321	vq = sc->vtballoon_inflate_vq;
322
323	if (npages > VTBALLOON_PAGES_PER_REQUEST)
324		npages = VTBALLOON_PAGES_PER_REQUEST;
325
326	for (i = 0; i < npages; i++) {
327		if ((m = vtballoon_alloc_page(sc)) == NULL) {
328			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
329			break;
330		}
331
332		sc->vtballoon_page_frames[i] =
333		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
334
335		KASSERT(m->queue == PQ_NONE,
336		    ("%s: allocated page %p on queue", __func__, m));
337		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
338	}
339
340	if (i > 0)
341		vtballoon_send_page_frames(sc, vq, i);
342}
343
344static void
345vtballoon_deflate(struct vtballoon_softc *sc, int npages)
346{
347	TAILQ_HEAD(, vm_page) free_pages;
348	struct virtqueue *vq;
349	vm_page_t m;
350	int i;
351
352	vq = sc->vtballoon_deflate_vq;
353	TAILQ_INIT(&free_pages);
354
355	if (npages > VTBALLOON_PAGES_PER_REQUEST)
356		npages = VTBALLOON_PAGES_PER_REQUEST;
357
358	for (i = 0; i < npages; i++) {
359		m = TAILQ_FIRST(&sc->vtballoon_pages);
360		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
361
362		sc->vtballoon_page_frames[i] =
363		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
364
365		TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
366		TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
367	}
368
369	if (i > 0) {
370		/* Always tell host first before freeing the pages. */
371		vtballoon_send_page_frames(sc, vq, i);
372
373		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
374			TAILQ_REMOVE(&free_pages, m, plinks.q);
375			vtballoon_free_page(sc, m);
376		}
377	}
378
379	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
380	    sc->vtballoon_current_npages == 0) ||
381	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
382	    sc->vtballoon_current_npages != 0),
383	    ("%s: bogus page count %d", __func__,
384	    sc->vtballoon_current_npages));
385}
386
387static void
388vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
389    int npages)
390{
391	struct sglist sg;
392	struct sglist_seg segs[1];
393	void *c;
394	int error;
395
396	sglist_init(&sg, 1, segs);
397
398	error = sglist_append(&sg, sc->vtballoon_page_frames,
399	    npages * sizeof(uint32_t));
400	KASSERT(error == 0, ("error adding page frames to sglist"));
401
402	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
403	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
404	virtqueue_notify(vq);
405
406	/*
407	 * Inflate and deflate operations are done synchronously. The
408	 * interrupt handler will wake us up.
409	 */
410	VTBALLOON_LOCK(sc);
411	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
412		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
413	VTBALLOON_UNLOCK(sc);
414
415	KASSERT(c == vq, ("unexpected balloon operation response"));
416}
417
418static void
419vtballoon_pop(struct vtballoon_softc *sc)
420{
421
422	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
423		vtballoon_deflate(sc, sc->vtballoon_current_npages);
424}
425
426static void
427vtballoon_stop(struct vtballoon_softc *sc)
428{
429
430	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
431	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
432
433	virtio_stop(sc->vtballoon_dev);
434}
435
436static vm_page_t
437vtballoon_alloc_page(struct vtballoon_softc *sc)
438{
439	vm_page_t m;
440
441	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
442	if (m != NULL)
443		sc->vtballoon_current_npages++;
444
445	return (m);
446}
447
448static void
449vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
450{
451
452	vm_page_free(m);
453	sc->vtballoon_current_npages--;
454}
455
456static uint32_t
457vtballoon_desired_size(struct vtballoon_softc *sc)
458{
459	uint32_t desired;
460
461	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
462	    offsetof(struct virtio_balloon_config, num_pages));
463
464	return (le32toh(desired));
465}
466
467static void
468vtballoon_update_size(struct vtballoon_softc *sc)
469{
470
471	virtio_write_dev_config_4(sc->vtballoon_dev,
472	    offsetof(struct virtio_balloon_config, actual),
473	    htole32(sc->vtballoon_current_npages));
474}
475
476static int
477vtballoon_sleep(struct vtballoon_softc *sc)
478{
479	int rc, timeout;
480	uint32_t current, desired;
481
482	rc = 0;
483	current = sc->vtballoon_current_npages;
484
485	VTBALLOON_LOCK(sc);
486	for (;;) {
487		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
488			rc = 1;
489			break;
490		}
491
492		desired = vtballoon_desired_size(sc);
493		sc->vtballoon_desired_npages = desired;
494
495		/*
496		 * If given, use non-zero timeout on the first time through
497		 * the loop. On subsequent times, timeout will be zero so
498		 * we will reevaluate the desired size of the balloon and
499		 * break out to retry if needed.
500		 */
501		timeout = sc->vtballoon_timeout;
502		sc->vtballoon_timeout = 0;
503
504		if (current > desired)
505			break;
506		if (current < desired && timeout == 0)
507			break;
508
509		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
510	}
511	VTBALLOON_UNLOCK(sc);
512
513	return (rc);
514}
515
516static void
517vtballoon_thread(void *xsc)
518{
519	struct vtballoon_softc *sc;
520	uint32_t current, desired;
521
522	sc = xsc;
523
524	for (;;) {
525		if (vtballoon_sleep(sc) != 0)
526			break;
527
528		current = sc->vtballoon_current_npages;
529		desired = sc->vtballoon_desired_npages;
530
531		if (desired != current) {
532			if (desired > current)
533				vtballoon_inflate(sc, desired - current);
534			else
535				vtballoon_deflate(sc, current - desired);
536
537			vtballoon_update_size(sc);
538		}
539	}
540
541	kthread_exit();
542}
543
544static void
545vtballoon_add_sysctl(struct vtballoon_softc *sc)
546{
547	device_t dev;
548	struct sysctl_ctx_list *ctx;
549	struct sysctl_oid *tree;
550	struct sysctl_oid_list *child;
551
552	dev = sc->vtballoon_dev;
553	ctx = device_get_sysctl_ctx(dev);
554	tree = device_get_sysctl_tree(dev);
555	child = SYSCTL_CHILDREN(tree);
556
557	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
558	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
559	    "Desired balloon size in pages");
560
561	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
562	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
563	    "Current balloon size in pages");
564}
565