1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice unmodified, this list of conditions, and the following
12 *    disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/* Driver for VirtIO memory balloon devices. */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD$");
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/kernel.h>
37#include <sys/endian.h>
38#include <sys/kthread.h>
39#include <sys/malloc.h>
40#include <sys/module.h>
41#include <sys/sglist.h>
42#include <sys/sysctl.h>
43#include <sys/lock.h>
44#include <sys/mutex.h>
45#include <sys/queue.h>
46
47#include <vm/vm.h>
48#include <vm/vm_page.h>
49
50#include <machine/bus.h>
51#include <machine/resource.h>
52#include <sys/bus.h>
53#include <sys/rman.h>
54
55#include <dev/virtio/virtio.h>
56#include <dev/virtio/virtqueue.h>
57#include <dev/virtio/balloon/virtio_balloon.h>
58
59#include "virtio_if.h"
60
61struct vtballoon_softc {
62	device_t		 vtballoon_dev;
63	struct mtx		 vtballoon_mtx;
64	uint64_t		 vtballoon_features;
65	uint32_t		 vtballoon_flags;
66#define VTBALLOON_FLAG_DETACH	 0x01
67
68	struct virtqueue	*vtballoon_inflate_vq;
69	struct virtqueue	*vtballoon_deflate_vq;
70
71	uint32_t		 vtballoon_desired_npages;
72	uint32_t		 vtballoon_current_npages;
73	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
74
75	struct thread		*vtballoon_td;
76	uint32_t		*vtballoon_page_frames;
77	int			 vtballoon_timeout;
78};
79
80static struct virtio_feature_desc vtballoon_feature_desc[] = {
81	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
82	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
83	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM,	"DeflateOnOOM"	},
84
85	{ 0, NULL }
86};
87
88static int	vtballoon_probe(device_t);
89static int	vtballoon_attach(device_t);
90static int	vtballoon_detach(device_t);
91static int	vtballoon_config_change(device_t);
92
93static int	vtballoon_negotiate_features(struct vtballoon_softc *);
94static int	vtballoon_setup_features(struct vtballoon_softc *);
95static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
96
97static void	vtballoon_vq_intr(void *);
98
99static void	vtballoon_inflate(struct vtballoon_softc *, int);
100static void	vtballoon_deflate(struct vtballoon_softc *, int);
101
102static void	vtballoon_send_page_frames(struct vtballoon_softc *,
103		    struct virtqueue *, int);
104
105static void	vtballoon_pop(struct vtballoon_softc *);
106static void	vtballoon_stop(struct vtballoon_softc *);
107
108static vm_page_t
109		vtballoon_alloc_page(struct vtballoon_softc *);
110static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
111
112static int	vtballoon_sleep(struct vtballoon_softc *);
113static void	vtballoon_thread(void *);
114static void	vtballoon_setup_sysctl(struct vtballoon_softc *);
115
116#define vtballoon_modern(_sc) \
117    (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
118
119/* Features desired/implemented by this driver. */
120#define VTBALLOON_FEATURES		VIRTIO_BALLOON_F_MUST_TELL_HOST
121
122/* Timeout between retries when the balloon needs inflating. */
123#define VTBALLOON_LOWMEM_TIMEOUT	hz
124
125/*
126 * Maximum number of pages we'll request to inflate or deflate
127 * the balloon in one virtqueue request. Both Linux and NetBSD
128 * have settled on 256, doing up to 1MB at a time.
129 */
130#define VTBALLOON_PAGES_PER_REQUEST	256
131
132/* Must be able to fix all pages frames in one page (segment). */
133CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
134
135#define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
136#define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
137					    "VirtIO Balloon Lock", MTX_DEF)
138#define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
139#define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
140#define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
141
142static device_method_t vtballoon_methods[] = {
143	/* Device methods. */
144	DEVMETHOD(device_probe,		vtballoon_probe),
145	DEVMETHOD(device_attach,	vtballoon_attach),
146	DEVMETHOD(device_detach,	vtballoon_detach),
147
148	/* VirtIO methods. */
149	DEVMETHOD(virtio_config_change, vtballoon_config_change),
150
151	DEVMETHOD_END
152};
153
154static driver_t vtballoon_driver = {
155	"vtballoon",
156	vtballoon_methods,
157	sizeof(struct vtballoon_softc)
158};
159static devclass_t vtballoon_devclass;
160
161VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver,
162    vtballoon_devclass, 0, 0);
163MODULE_VERSION(virtio_balloon, 1);
164MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
165
166VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
167    "VirtIO Balloon Adapter");
168
169static int
170vtballoon_probe(device_t dev)
171{
172	return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
173}
174
175static int
176vtballoon_attach(device_t dev)
177{
178	struct vtballoon_softc *sc;
179	int error;
180
181	sc = device_get_softc(dev);
182	sc->vtballoon_dev = dev;
183	virtio_set_feature_desc(dev, vtballoon_feature_desc);
184
185	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
186	TAILQ_INIT(&sc->vtballoon_pages);
187
188	vtballoon_setup_sysctl(sc);
189
190	error = vtballoon_setup_features(sc);
191	if (error) {
192		device_printf(dev, "cannot setup features\n");
193		goto fail;
194	}
195
196	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
197	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
198	if (sc->vtballoon_page_frames == NULL) {
199		error = ENOMEM;
200		device_printf(dev,
201		    "cannot allocate page frame request array\n");
202		goto fail;
203	}
204
205	error = vtballoon_alloc_virtqueues(sc);
206	if (error) {
207		device_printf(dev, "cannot allocate virtqueues\n");
208		goto fail;
209	}
210
211	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
212	if (error) {
213		device_printf(dev, "cannot setup virtqueue interrupts\n");
214		goto fail;
215	}
216
217	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
218	    0, 0, "virtio_balloon");
219	if (error) {
220		device_printf(dev, "cannot create balloon kthread\n");
221		goto fail;
222	}
223
224	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
225	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
226
227fail:
228	if (error)
229		vtballoon_detach(dev);
230
231	return (error);
232}
233
234static int
235vtballoon_detach(device_t dev)
236{
237	struct vtballoon_softc *sc;
238
239	sc = device_get_softc(dev);
240
241	if (sc->vtballoon_td != NULL) {
242		VTBALLOON_LOCK(sc);
243		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
244		wakeup_one(sc);
245		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
246		VTBALLOON_UNLOCK(sc);
247
248		sc->vtballoon_td = NULL;
249	}
250
251	if (device_is_attached(dev)) {
252		vtballoon_pop(sc);
253		vtballoon_stop(sc);
254	}
255
256	if (sc->vtballoon_page_frames != NULL) {
257		free(sc->vtballoon_page_frames, M_DEVBUF);
258		sc->vtballoon_page_frames = NULL;
259	}
260
261	VTBALLOON_LOCK_DESTROY(sc);
262
263	return (0);
264}
265
266static int
267vtballoon_config_change(device_t dev)
268{
269	struct vtballoon_softc *sc;
270
271	sc = device_get_softc(dev);
272
273	VTBALLOON_LOCK(sc);
274	wakeup_one(sc);
275	VTBALLOON_UNLOCK(sc);
276
277	return (1);
278}
279
280static int
281vtballoon_negotiate_features(struct vtballoon_softc *sc)
282{
283	device_t dev;
284	uint64_t features;
285
286	dev = sc->vtballoon_dev;
287	features = VTBALLOON_FEATURES;
288
289	sc->vtballoon_features = virtio_negotiate_features(dev, features);
290	return (virtio_finalize_features(dev));
291}
292
293static int
294vtballoon_setup_features(struct vtballoon_softc *sc)
295{
296	int error;
297
298	error = vtballoon_negotiate_features(sc);
299	if (error)
300		return (error);
301
302	return (0);
303}
304
305static int
306vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
307{
308	device_t dev;
309	struct vq_alloc_info vq_info[2];
310	int nvqs;
311
312	dev = sc->vtballoon_dev;
313	nvqs = 2;
314
315	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
316	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
317
318	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
319	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
320
321	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
322}
323
324static void
325vtballoon_vq_intr(void *xsc)
326{
327	struct vtballoon_softc *sc;
328
329	sc = xsc;
330
331	VTBALLOON_LOCK(sc);
332	wakeup_one(sc);
333	VTBALLOON_UNLOCK(sc);
334}
335
336static void
337vtballoon_inflate(struct vtballoon_softc *sc, int npages)
338{
339	struct virtqueue *vq;
340	vm_page_t m;
341	int i;
342
343	vq = sc->vtballoon_inflate_vq;
344
345	if (npages > VTBALLOON_PAGES_PER_REQUEST)
346		npages = VTBALLOON_PAGES_PER_REQUEST;
347
348	for (i = 0; i < npages; i++) {
349		if ((m = vtballoon_alloc_page(sc)) == NULL) {
350			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
351			break;
352		}
353
354		sc->vtballoon_page_frames[i] =
355		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
356
357		KASSERT(m->a.queue == PQ_NONE,
358		    ("%s: allocated page %p on queue", __func__, m));
359		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
360	}
361
362	if (i > 0)
363		vtballoon_send_page_frames(sc, vq, i);
364}
365
366static void
367vtballoon_deflate(struct vtballoon_softc *sc, int npages)
368{
369	TAILQ_HEAD(, vm_page) free_pages;
370	struct virtqueue *vq;
371	vm_page_t m;
372	int i;
373
374	vq = sc->vtballoon_deflate_vq;
375	TAILQ_INIT(&free_pages);
376
377	if (npages > VTBALLOON_PAGES_PER_REQUEST)
378		npages = VTBALLOON_PAGES_PER_REQUEST;
379
380	for (i = 0; i < npages; i++) {
381		m = TAILQ_FIRST(&sc->vtballoon_pages);
382		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
383
384		sc->vtballoon_page_frames[i] =
385		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
386
387		TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
388		TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
389	}
390
391	if (i > 0) {
392		/* Always tell host first before freeing the pages. */
393		vtballoon_send_page_frames(sc, vq, i);
394
395		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
396			TAILQ_REMOVE(&free_pages, m, plinks.q);
397			vtballoon_free_page(sc, m);
398		}
399	}
400
401	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
402	    sc->vtballoon_current_npages == 0) ||
403	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
404	    sc->vtballoon_current_npages != 0),
405	    ("%s: bogus page count %d", __func__,
406	    sc->vtballoon_current_npages));
407}
408
409static void
410vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
411    int npages)
412{
413	struct sglist sg;
414	struct sglist_seg segs[1];
415	void *c;
416	int error;
417
418	sglist_init(&sg, 1, segs);
419
420	error = sglist_append(&sg, sc->vtballoon_page_frames,
421	    npages * sizeof(uint32_t));
422	KASSERT(error == 0, ("error adding page frames to sglist"));
423
424	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
425	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
426	virtqueue_notify(vq);
427
428	/*
429	 * Inflate and deflate operations are done synchronously. The
430	 * interrupt handler will wake us up.
431	 */
432	VTBALLOON_LOCK(sc);
433	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
434		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
435	VTBALLOON_UNLOCK(sc);
436
437	KASSERT(c == vq, ("unexpected balloon operation response"));
438}
439
440static void
441vtballoon_pop(struct vtballoon_softc *sc)
442{
443
444	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
445		vtballoon_deflate(sc, sc->vtballoon_current_npages);
446}
447
448static void
449vtballoon_stop(struct vtballoon_softc *sc)
450{
451
452	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
453	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
454
455	virtio_stop(sc->vtballoon_dev);
456}
457
458static vm_page_t
459vtballoon_alloc_page(struct vtballoon_softc *sc)
460{
461	vm_page_t m;
462
463	m = vm_page_alloc(NULL, 0,
464	    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP);
465	if (m != NULL)
466		sc->vtballoon_current_npages++;
467
468	return (m);
469}
470
471static void
472vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
473{
474
475	vm_page_free(m);
476	sc->vtballoon_current_npages--;
477}
478
479static uint32_t
480vtballoon_desired_size(struct vtballoon_softc *sc)
481{
482	uint32_t desired;
483
484	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
485	    offsetof(struct virtio_balloon_config, num_pages));
486
487	if (vtballoon_modern(sc))
488		return (desired);
489	else
490		return (le32toh(desired));
491}
492
493static void
494vtballoon_update_size(struct vtballoon_softc *sc)
495{
496	uint32_t npages;
497
498	npages = sc->vtballoon_current_npages;
499	if (!vtballoon_modern(sc))
500		npages = htole32(npages);
501
502	virtio_write_dev_config_4(sc->vtballoon_dev,
503	    offsetof(struct virtio_balloon_config, actual), npages);
504}
505
506static int
507vtballoon_sleep(struct vtballoon_softc *sc)
508{
509	int rc, timeout;
510	uint32_t current, desired;
511
512	rc = 0;
513	current = sc->vtballoon_current_npages;
514
515	VTBALLOON_LOCK(sc);
516	for (;;) {
517		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
518			rc = 1;
519			break;
520		}
521
522		desired = vtballoon_desired_size(sc);
523		sc->vtballoon_desired_npages = desired;
524
525		/*
526		 * If given, use non-zero timeout on the first time through
527		 * the loop. On subsequent times, timeout will be zero so
528		 * we will reevaluate the desired size of the balloon and
529		 * break out to retry if needed.
530		 */
531		timeout = sc->vtballoon_timeout;
532		sc->vtballoon_timeout = 0;
533
534		if (current > desired)
535			break;
536		if (current < desired && timeout == 0)
537			break;
538
539		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
540	}
541	VTBALLOON_UNLOCK(sc);
542
543	return (rc);
544}
545
546static void
547vtballoon_thread(void *xsc)
548{
549	struct vtballoon_softc *sc;
550	uint32_t current, desired;
551
552	sc = xsc;
553
554	for (;;) {
555		if (vtballoon_sleep(sc) != 0)
556			break;
557
558		current = sc->vtballoon_current_npages;
559		desired = sc->vtballoon_desired_npages;
560
561		if (desired != current) {
562			if (desired > current)
563				vtballoon_inflate(sc, desired - current);
564			else
565				vtballoon_deflate(sc, current - desired);
566
567			vtballoon_update_size(sc);
568		}
569	}
570
571	kthread_exit();
572}
573
574static void
575vtballoon_setup_sysctl(struct vtballoon_softc *sc)
576{
577	device_t dev;
578	struct sysctl_ctx_list *ctx;
579	struct sysctl_oid *tree;
580	struct sysctl_oid_list *child;
581
582	dev = sc->vtballoon_dev;
583	ctx = device_get_sysctl_ctx(dev);
584	tree = device_get_sysctl_tree(dev);
585	child = SYSCTL_CHILDREN(tree);
586
587	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
588	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
589	    "Desired balloon size in pages");
590
591	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
592	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
593	    "Current balloon size in pages");
594}
595