1// SPDX-License-Identifier: GPL-2.0
2/*
3 * DAMON-based page reclamation
4 *
5 * Author: SeongJae Park <sj@kernel.org>
6 */
7
8#define pr_fmt(fmt) "damon-reclaim: " fmt
9
10#include <linux/damon.h>
11#include <linux/kstrtox.h>
12#include <linux/module.h>
13
14#include "modules-common.h"
15
16#ifdef MODULE_PARAM_PREFIX
17#undef MODULE_PARAM_PREFIX
18#endif
19#define MODULE_PARAM_PREFIX "damon_reclaim."
20
21/*
22 * Enable or disable DAMON_RECLAIM.
23 *
24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
25 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
26 * do no real monitoring and reclamation due to the watermarks-based activation
27 * condition.  Refer to below descriptions for the watermarks parameter for
28 * this.
29 */
30static bool enabled __read_mostly;
31
32/*
33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
34 *
35 * Input parameters that updated while DAMON_RECLAIM is running are not applied
36 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
37 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
38 * parameter is set as ``N``.  If invalid parameters are found while the
39 * re-reading, DAMON_RECLAIM will be disabled.
40 */
41static bool commit_inputs __read_mostly;
42module_param(commit_inputs, bool, 0600);
43
44/*
45 * Time threshold for cold memory regions identification in microseconds.
46 *
47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
48 * identifies the region as cold, and reclaims.  120 seconds by default.
49 */
50static unsigned long min_age __read_mostly = 120000000;
51module_param(min_age, ulong, 0600);
52
53static struct damos_quota damon_reclaim_quota = {
54	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
55	.ms = 10,
56	.sz = 128 * 1024 * 1024,
57	.reset_interval = 1000,
58	/* Within the quota, page out older regions first. */
59	.weight_sz = 0,
60	.weight_nr_accesses = 0,
61	.weight_age = 1
62};
63DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
64
65/*
66 * Desired level of memory pressure-stall time in microseconds.
67 *
68 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
69 * increases and decreases the effective level of the quota aiming this level of
70 * memory pressure is incurred.  System-wide ``some`` memory PSI in microseconds
71 * per quota reset interval (``quota_reset_interval_ms``) is collected and
72 * compared to this value to see if the aim is satisfied.  Value zero means
73 * disabling this auto-tuning feature.
74 *
75 * Disabled by default.
76 */
77static unsigned long quota_mem_pressure_us __read_mostly;
78module_param(quota_mem_pressure_us, ulong, 0600);
79
80/*
81 * User-specifiable feedback for auto-tuning of the effective quota.
82 *
83 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
84 * increases and decreases the effective level of the quota aiming receiving this
85 * feedback of value ``10,000`` from the user.  DAMON_RECLAIM assumes the feedback
86 * value and the quota are positively proportional.  Value zero means disabling
87 * this auto-tuning feature.
88 *
89 * Disabled by default.
90 *
91 */
92static unsigned long quota_autotune_feedback __read_mostly;
93module_param(quota_autotune_feedback, ulong, 0600);
94
95static struct damos_watermarks damon_reclaim_wmarks = {
96	.metric = DAMOS_WMARK_FREE_MEM_RATE,
97	.interval = 5000000,	/* 5 seconds */
98	.high = 500,		/* 50 percent */
99	.mid = 400,		/* 40 percent */
100	.low = 200,		/* 20 percent */
101};
102DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
103
104static struct damon_attrs damon_reclaim_mon_attrs = {
105	.sample_interval = 5000,	/* 5 ms */
106	.aggr_interval = 100000,	/* 100 ms */
107	.ops_update_interval = 0,
108	.min_nr_regions = 10,
109	.max_nr_regions = 1000,
110};
111DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
112
113/*
114 * Start of the target memory region in physical address.
115 *
116 * The start physical address of memory region that DAMON_RECLAIM will do work
117 * against.  By default, biggest System RAM is used as the region.
118 */
119static unsigned long monitor_region_start __read_mostly;
120module_param(monitor_region_start, ulong, 0600);
121
122/*
123 * End of the target memory region in physical address.
124 *
125 * The end physical address of memory region that DAMON_RECLAIM will do work
126 * against.  By default, biggest System RAM is used as the region.
127 */
128static unsigned long monitor_region_end __read_mostly;
129module_param(monitor_region_end, ulong, 0600);
130
131/*
132 * Skip anonymous pages reclamation.
133 *
134 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous
135 * pages.  By default, ``N``.
136 */
137static bool skip_anon __read_mostly;
138module_param(skip_anon, bool, 0600);
139
140/*
141 * PID of the DAMON thread
142 *
143 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
144 * Else, -1.
145 */
146static int kdamond_pid __read_mostly = -1;
147module_param(kdamond_pid, int, 0400);
148
149static struct damos_stat damon_reclaim_stat;
150DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
151		reclaim_tried_regions, reclaimed_regions, quota_exceeds);
152
153static struct damon_ctx *ctx;
154static struct damon_target *target;
155
156static struct damos *damon_reclaim_new_scheme(void)
157{
158	struct damos_access_pattern pattern = {
159		/* Find regions having PAGE_SIZE or larger size */
160		.min_sz_region = PAGE_SIZE,
161		.max_sz_region = ULONG_MAX,
162		/* and not accessed at all */
163		.min_nr_accesses = 0,
164		.max_nr_accesses = 0,
165		/* for min_age or more micro-seconds */
166		.min_age_region = min_age /
167			damon_reclaim_mon_attrs.aggr_interval,
168		.max_age_region = UINT_MAX,
169	};
170
171	return damon_new_scheme(
172			&pattern,
173			/* page out those, as soon as found */
174			DAMOS_PAGEOUT,
175			/* for each aggregation interval */
176			0,
177			/* under the quota. */
178			&damon_reclaim_quota,
179			/* (De)activate this according to the watermarks. */
180			&damon_reclaim_wmarks);
181}
182
183static void damon_reclaim_copy_quota_status(struct damos_quota *dst,
184		struct damos_quota *src)
185{
186	dst->total_charged_sz = src->total_charged_sz;
187	dst->total_charged_ns = src->total_charged_ns;
188	dst->charged_sz = src->charged_sz;
189	dst->charged_from = src->charged_from;
190	dst->charge_target_from = src->charge_target_from;
191	dst->charge_addr_from = src->charge_addr_from;
192	dst->esz_bp = src->esz_bp;
193}
194
195static int damon_reclaim_apply_parameters(void)
196{
197	struct damos *scheme, *old_scheme;
198	struct damos_quota_goal *goal;
199	struct damos_filter *filter;
200	int err = 0;
201
202	err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
203	if (err)
204		return err;
205
206	/* Will be freed by next 'damon_set_schemes()' below */
207	scheme = damon_reclaim_new_scheme();
208	if (!scheme)
209		return -ENOMEM;
210	if (!list_empty(&ctx->schemes)) {
211		damon_for_each_scheme(old_scheme, ctx)
212			damon_reclaim_copy_quota_status(&scheme->quota,
213					&old_scheme->quota);
214	}
215
216	if (quota_mem_pressure_us) {
217		goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US,
218				quota_mem_pressure_us);
219		if (!goal) {
220			damon_destroy_scheme(scheme);
221			return -ENOMEM;
222		}
223		damos_add_quota_goal(&scheme->quota, goal);
224	}
225
226	if (quota_autotune_feedback) {
227		goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000);
228		if (!goal) {
229			damon_destroy_scheme(scheme);
230			return -ENOMEM;
231		}
232		goal->current_value = quota_autotune_feedback;
233		damos_add_quota_goal(&scheme->quota, goal);
234	}
235
236	if (skip_anon) {
237		filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true);
238		if (!filter) {
239			/* Will be freed by next 'damon_set_schemes()' below */
240			damon_destroy_scheme(scheme);
241			return -ENOMEM;
242		}
243		damos_add_filter(scheme, filter);
244	}
245	damon_set_schemes(ctx, &scheme, 1);
246
247	return damon_set_region_biggest_system_ram_default(target,
248					&monitor_region_start,
249					&monitor_region_end);
250}
251
252static int damon_reclaim_turn(bool on)
253{
254	int err;
255
256	if (!on) {
257		err = damon_stop(&ctx, 1);
258		if (!err)
259			kdamond_pid = -1;
260		return err;
261	}
262
263	err = damon_reclaim_apply_parameters();
264	if (err)
265		return err;
266
267	err = damon_start(&ctx, 1, true);
268	if (err)
269		return err;
270	kdamond_pid = ctx->kdamond->pid;
271	return 0;
272}
273
274static int damon_reclaim_enabled_store(const char *val,
275		const struct kernel_param *kp)
276{
277	bool is_enabled = enabled;
278	bool enable;
279	int err;
280
281	err = kstrtobool(val, &enable);
282	if (err)
283		return err;
284
285	if (is_enabled == enable)
286		return 0;
287
288	/* Called before init function.  The function will handle this. */
289	if (!ctx)
290		goto set_param_out;
291
292	err = damon_reclaim_turn(enable);
293	if (err)
294		return err;
295
296set_param_out:
297	enabled = enable;
298	return err;
299}
300
301static const struct kernel_param_ops enabled_param_ops = {
302	.set = damon_reclaim_enabled_store,
303	.get = param_get_bool,
304};
305
306module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
307MODULE_PARM_DESC(enabled,
308	"Enable or disable DAMON_RECLAIM (default: disabled)");
309
310static int damon_reclaim_handle_commit_inputs(void)
311{
312	int err;
313
314	if (!commit_inputs)
315		return 0;
316
317	err = damon_reclaim_apply_parameters();
318	commit_inputs = false;
319	return err;
320}
321
322static int damon_reclaim_after_aggregation(struct damon_ctx *c)
323{
324	struct damos *s;
325
326	/* update the stats parameter */
327	damon_for_each_scheme(s, c)
328		damon_reclaim_stat = s->stat;
329
330	return damon_reclaim_handle_commit_inputs();
331}
332
333static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
334{
335	return damon_reclaim_handle_commit_inputs();
336}
337
338static int __init damon_reclaim_init(void)
339{
340	int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
341
342	if (err)
343		return err;
344
345	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
346	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
347
348	/* 'enabled' has set before this function, probably via command line */
349	if (enabled)
350		err = damon_reclaim_turn(true);
351
352	return err;
353}
354
355module_init(damon_reclaim_init);
356