1// SPDX-License-Identifier: GPL-2.0 2/* 3 * DAMON-based page reclamation 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8#define pr_fmt(fmt) "damon-reclaim: " fmt 9 10#include <linux/damon.h> 11#include <linux/kstrtox.h> 12#include <linux/module.h> 13 14#include "modules-common.h" 15 16#ifdef MODULE_PARAM_PREFIX 17#undef MODULE_PARAM_PREFIX 18#endif 19#define MODULE_PARAM_PREFIX "damon_reclaim." 20 21/* 22 * Enable or disable DAMON_RECLAIM. 23 * 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. 25 * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could 26 * do no real monitoring and reclamation due to the watermarks-based activation 27 * condition. Refer to below descriptions for the watermarks parameter for 28 * this. 29 */ 30static bool enabled __read_mostly; 31 32/* 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. 34 * 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied 36 * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values 37 * of parametrs except ``enabled`` again. Once the re-reading is done, this 38 * parameter is set as ``N``. If invalid parameters are found while the 39 * re-reading, DAMON_RECLAIM will be disabled. 40 */ 41static bool commit_inputs __read_mostly; 42module_param(commit_inputs, bool, 0600); 43 44/* 45 * Time threshold for cold memory regions identification in microseconds. 46 * 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM 48 * identifies the region as cold, and reclaims. 120 seconds by default. 49 */ 50static unsigned long min_age __read_mostly = 120000000; 51module_param(min_age, ulong, 0600); 52 53static struct damos_quota damon_reclaim_quota = { 54 /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ 55 .ms = 10, 56 .sz = 128 * 1024 * 1024, 57 .reset_interval = 1000, 58 /* Within the quota, page out older regions first. */ 59 .weight_sz = 0, 60 .weight_nr_accesses = 0, 61 .weight_age = 1 62}; 63DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); 64 65/* 66 * Desired level of memory pressure-stall time in microseconds. 67 * 68 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically 69 * increases and decreases the effective level of the quota aiming this level of 70 * memory pressure is incurred. System-wide ``some`` memory PSI in microseconds 71 * per quota reset interval (``quota_reset_interval_ms``) is collected and 72 * compared to this value to see if the aim is satisfied. Value zero means 73 * disabling this auto-tuning feature. 74 * 75 * Disabled by default. 76 */ 77static unsigned long quota_mem_pressure_us __read_mostly; 78module_param(quota_mem_pressure_us, ulong, 0600); 79 80/* 81 * User-specifiable feedback for auto-tuning of the effective quota. 82 * 83 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically 84 * increases and decreases the effective level of the quota aiming receiving this 85 * feedback of value ``10,000`` from the user. DAMON_RECLAIM assumes the feedback 86 * value and the quota are positively proportional. Value zero means disabling 87 * this auto-tuning feature. 88 * 89 * Disabled by default. 90 * 91 */ 92static unsigned long quota_autotune_feedback __read_mostly; 93module_param(quota_autotune_feedback, ulong, 0600); 94 95static struct damos_watermarks damon_reclaim_wmarks = { 96 .metric = DAMOS_WMARK_FREE_MEM_RATE, 97 .interval = 5000000, /* 5 seconds */ 98 .high = 500, /* 50 percent */ 99 .mid = 400, /* 40 percent */ 100 .low = 200, /* 20 percent */ 101}; 102DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); 103 104static struct damon_attrs damon_reclaim_mon_attrs = { 105 .sample_interval = 5000, /* 5 ms */ 106 .aggr_interval = 100000, /* 100 ms */ 107 .ops_update_interval = 0, 108 .min_nr_regions = 10, 109 .max_nr_regions = 1000, 110}; 111DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); 112 113/* 114 * Start of the target memory region in physical address. 115 * 116 * The start physical address of memory region that DAMON_RECLAIM will do work 117 * against. By default, biggest System RAM is used as the region. 118 */ 119static unsigned long monitor_region_start __read_mostly; 120module_param(monitor_region_start, ulong, 0600); 121 122/* 123 * End of the target memory region in physical address. 124 * 125 * The end physical address of memory region that DAMON_RECLAIM will do work 126 * against. By default, biggest System RAM is used as the region. 127 */ 128static unsigned long monitor_region_end __read_mostly; 129module_param(monitor_region_end, ulong, 0600); 130 131/* 132 * Skip anonymous pages reclamation. 133 * 134 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous 135 * pages. By default, ``N``. 136 */ 137static bool skip_anon __read_mostly; 138module_param(skip_anon, bool, 0600); 139 140/* 141 * PID of the DAMON thread 142 * 143 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. 144 * Else, -1. 145 */ 146static int kdamond_pid __read_mostly = -1; 147module_param(kdamond_pid, int, 0400); 148 149static struct damos_stat damon_reclaim_stat; 150DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, 151 reclaim_tried_regions, reclaimed_regions, quota_exceeds); 152 153static struct damon_ctx *ctx; 154static struct damon_target *target; 155 156static struct damos *damon_reclaim_new_scheme(void) 157{ 158 struct damos_access_pattern pattern = { 159 /* Find regions having PAGE_SIZE or larger size */ 160 .min_sz_region = PAGE_SIZE, 161 .max_sz_region = ULONG_MAX, 162 /* and not accessed at all */ 163 .min_nr_accesses = 0, 164 .max_nr_accesses = 0, 165 /* for min_age or more micro-seconds */ 166 .min_age_region = min_age / 167 damon_reclaim_mon_attrs.aggr_interval, 168 .max_age_region = UINT_MAX, 169 }; 170 171 return damon_new_scheme( 172 &pattern, 173 /* page out those, as soon as found */ 174 DAMOS_PAGEOUT, 175 /* for each aggregation interval */ 176 0, 177 /* under the quota. */ 178 &damon_reclaim_quota, 179 /* (De)activate this according to the watermarks. */ 180 &damon_reclaim_wmarks); 181} 182 183static void damon_reclaim_copy_quota_status(struct damos_quota *dst, 184 struct damos_quota *src) 185{ 186 dst->total_charged_sz = src->total_charged_sz; 187 dst->total_charged_ns = src->total_charged_ns; 188 dst->charged_sz = src->charged_sz; 189 dst->charged_from = src->charged_from; 190 dst->charge_target_from = src->charge_target_from; 191 dst->charge_addr_from = src->charge_addr_from; 192 dst->esz_bp = src->esz_bp; 193} 194 195static int damon_reclaim_apply_parameters(void) 196{ 197 struct damos *scheme, *old_scheme; 198 struct damos_quota_goal *goal; 199 struct damos_filter *filter; 200 int err = 0; 201 202 err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); 203 if (err) 204 return err; 205 206 /* Will be freed by next 'damon_set_schemes()' below */ 207 scheme = damon_reclaim_new_scheme(); 208 if (!scheme) 209 return -ENOMEM; 210 if (!list_empty(&ctx->schemes)) { 211 damon_for_each_scheme(old_scheme, ctx) 212 damon_reclaim_copy_quota_status(&scheme->quota, 213 &old_scheme->quota); 214 } 215 216 if (quota_mem_pressure_us) { 217 goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US, 218 quota_mem_pressure_us); 219 if (!goal) { 220 damon_destroy_scheme(scheme); 221 return -ENOMEM; 222 } 223 damos_add_quota_goal(&scheme->quota, goal); 224 } 225 226 if (quota_autotune_feedback) { 227 goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000); 228 if (!goal) { 229 damon_destroy_scheme(scheme); 230 return -ENOMEM; 231 } 232 goal->current_value = quota_autotune_feedback; 233 damos_add_quota_goal(&scheme->quota, goal); 234 } 235 236 if (skip_anon) { 237 filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true); 238 if (!filter) { 239 /* Will be freed by next 'damon_set_schemes()' below */ 240 damon_destroy_scheme(scheme); 241 return -ENOMEM; 242 } 243 damos_add_filter(scheme, filter); 244 } 245 damon_set_schemes(ctx, &scheme, 1); 246 247 return damon_set_region_biggest_system_ram_default(target, 248 &monitor_region_start, 249 &monitor_region_end); 250} 251 252static int damon_reclaim_turn(bool on) 253{ 254 int err; 255 256 if (!on) { 257 err = damon_stop(&ctx, 1); 258 if (!err) 259 kdamond_pid = -1; 260 return err; 261 } 262 263 err = damon_reclaim_apply_parameters(); 264 if (err) 265 return err; 266 267 err = damon_start(&ctx, 1, true); 268 if (err) 269 return err; 270 kdamond_pid = ctx->kdamond->pid; 271 return 0; 272} 273 274static int damon_reclaim_enabled_store(const char *val, 275 const struct kernel_param *kp) 276{ 277 bool is_enabled = enabled; 278 bool enable; 279 int err; 280 281 err = kstrtobool(val, &enable); 282 if (err) 283 return err; 284 285 if (is_enabled == enable) 286 return 0; 287 288 /* Called before init function. The function will handle this. */ 289 if (!ctx) 290 goto set_param_out; 291 292 err = damon_reclaim_turn(enable); 293 if (err) 294 return err; 295 296set_param_out: 297 enabled = enable; 298 return err; 299} 300 301static const struct kernel_param_ops enabled_param_ops = { 302 .set = damon_reclaim_enabled_store, 303 .get = param_get_bool, 304}; 305 306module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 307MODULE_PARM_DESC(enabled, 308 "Enable or disable DAMON_RECLAIM (default: disabled)"); 309 310static int damon_reclaim_handle_commit_inputs(void) 311{ 312 int err; 313 314 if (!commit_inputs) 315 return 0; 316 317 err = damon_reclaim_apply_parameters(); 318 commit_inputs = false; 319 return err; 320} 321 322static int damon_reclaim_after_aggregation(struct damon_ctx *c) 323{ 324 struct damos *s; 325 326 /* update the stats parameter */ 327 damon_for_each_scheme(s, c) 328 damon_reclaim_stat = s->stat; 329 330 return damon_reclaim_handle_commit_inputs(); 331} 332 333static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) 334{ 335 return damon_reclaim_handle_commit_inputs(); 336} 337 338static int __init damon_reclaim_init(void) 339{ 340 int err = damon_modules_new_paddr_ctx_target(&ctx, &target); 341 342 if (err) 343 return err; 344 345 ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; 346 ctx->callback.after_aggregation = damon_reclaim_after_aggregation; 347 348 /* 'enabled' has set before this function, probably via command line */ 349 if (enabled) 350 err = damon_reclaim_turn(true); 351 352 return err; 353} 354 355module_init(damon_reclaim_init); 356