1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include <linux/vmalloc.h>
8#include <linux/sched/mm.h>
9
10#include "msm_drv.h"
11#include "msm_gem.h"
12#include "msm_gpu.h"
13#include "msm_gpu_trace.h"
14
15/* Default disabled for now until it has some more testing on the different
16 * iommu combinations that can be paired with the driver:
17 */
18static bool enable_eviction = true;
19MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
20module_param(enable_eviction, bool, 0600);
21
22static bool can_swap(void)
23{
24	return enable_eviction && get_nr_swap_pages() > 0;
25}
26
27static bool can_block(struct shrink_control *sc)
28{
29	if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM))
30		return false;
31	return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM);
32}
33
34static unsigned long
35msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
36{
37	struct msm_drm_private *priv = shrinker->private_data;
38	unsigned count = priv->lru.dontneed.count;
39
40	if (can_swap())
41		count += priv->lru.willneed.count;
42
43	return count;
44}
45
46static bool
47purge(struct drm_gem_object *obj)
48{
49	if (!is_purgeable(to_msm_bo(obj)))
50		return false;
51
52	if (msm_gem_active(obj))
53		return false;
54
55	msm_gem_purge(obj);
56
57	return true;
58}
59
60static bool
61evict(struct drm_gem_object *obj)
62{
63	if (is_unevictable(to_msm_bo(obj)))
64		return false;
65
66	if (msm_gem_active(obj))
67		return false;
68
69	msm_gem_evict(obj);
70
71	return true;
72}
73
74static bool
75wait_for_idle(struct drm_gem_object *obj)
76{
77	enum dma_resv_usage usage = dma_resv_usage_rw(true);
78	return dma_resv_wait_timeout(obj->resv, usage, false, 10) > 0;
79}
80
81static bool
82active_purge(struct drm_gem_object *obj)
83{
84	if (!wait_for_idle(obj))
85		return false;
86
87	return purge(obj);
88}
89
90static bool
91active_evict(struct drm_gem_object *obj)
92{
93	if (!wait_for_idle(obj))
94		return false;
95
96	return evict(obj);
97}
98
99static unsigned long
100msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
101{
102	struct msm_drm_private *priv = shrinker->private_data;
103	struct {
104		struct drm_gem_lru *lru;
105		bool (*shrink)(struct drm_gem_object *obj);
106		bool cond;
107		unsigned long freed;
108		unsigned long remaining;
109	} stages[] = {
110		/* Stages of progressively more aggressive/expensive reclaim: */
111		{ &priv->lru.dontneed, purge,        true },
112		{ &priv->lru.willneed, evict,        can_swap() },
113		{ &priv->lru.dontneed, active_purge, can_block(sc) },
114		{ &priv->lru.willneed, active_evict, can_swap() && can_block(sc) },
115	};
116	long nr = sc->nr_to_scan;
117	unsigned long freed = 0;
118	unsigned long remaining = 0;
119
120	for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
121		if (!stages[i].cond)
122			continue;
123		stages[i].freed =
124			drm_gem_lru_scan(stages[i].lru, nr,
125					&stages[i].remaining,
126					 stages[i].shrink);
127		nr -= stages[i].freed;
128		freed += stages[i].freed;
129		remaining += stages[i].remaining;
130	}
131
132	if (freed) {
133		trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed,
134				     stages[1].freed, stages[2].freed,
135				     stages[3].freed);
136	}
137
138	return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP;
139}
140
141#ifdef CONFIG_DEBUG_FS
142unsigned long
143msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
144{
145	struct msm_drm_private *priv = dev->dev_private;
146	struct shrink_control sc = {
147		.nr_to_scan = nr_to_scan,
148	};
149	unsigned long ret = SHRINK_STOP;
150
151	fs_reclaim_acquire(GFP_KERNEL);
152	if (priv->shrinker)
153		ret = msm_gem_shrinker_scan(priv->shrinker, &sc);
154	fs_reclaim_release(GFP_KERNEL);
155
156	return ret;
157}
158#endif
159
160/* since we don't know any better, lets bail after a few
161 * and if necessary the shrinker will be invoked again.
162 * Seems better than unmapping *everything*
163 */
164static const int vmap_shrink_limit = 15;
165
166static bool
167vmap_shrink(struct drm_gem_object *obj)
168{
169	if (!is_vunmapable(to_msm_bo(obj)))
170		return false;
171
172	msm_gem_vunmap(obj);
173
174	return true;
175}
176
177static int
178msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
179{
180	struct msm_drm_private *priv =
181		container_of(nb, struct msm_drm_private, vmap_notifier);
182	struct drm_gem_lru *lrus[] = {
183		&priv->lru.dontneed,
184		&priv->lru.willneed,
185		&priv->lru.pinned,
186		NULL,
187	};
188	unsigned idx, unmapped = 0;
189	unsigned long remaining = 0;
190
191	for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
192		unmapped += drm_gem_lru_scan(lrus[idx],
193					     vmap_shrink_limit - unmapped,
194					     &remaining,
195					     vmap_shrink);
196	}
197
198	*(unsigned long *)ptr += unmapped;
199
200	if (unmapped > 0)
201		trace_msm_gem_purge_vmaps(unmapped);
202
203	return NOTIFY_DONE;
204}
205
206/**
207 * msm_gem_shrinker_init - Initialize msm shrinker
208 * @dev: drm device
209 *
210 * This function registers and sets up the msm shrinker.
211 */
212int msm_gem_shrinker_init(struct drm_device *dev)
213{
214	struct msm_drm_private *priv = dev->dev_private;
215
216	priv->shrinker = shrinker_alloc(0, "drm-msm_gem");
217	if (!priv->shrinker)
218		return -ENOMEM;
219
220	priv->shrinker->count_objects = msm_gem_shrinker_count;
221	priv->shrinker->scan_objects = msm_gem_shrinker_scan;
222	priv->shrinker->private_data = priv;
223
224	shrinker_register(priv->shrinker);
225
226	priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
227	WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
228
229	return 0;
230}
231
232/**
233 * msm_gem_shrinker_cleanup - Clean up msm shrinker
234 * @dev: drm device
235 *
236 * This function unregisters the msm shrinker.
237 */
238void msm_gem_shrinker_cleanup(struct drm_device *dev)
239{
240	struct msm_drm_private *priv = dev->dev_private;
241
242	if (priv->shrinker) {
243		WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
244		shrinker_free(priv->shrinker);
245	}
246}
247