Lines Matching defs:memcg
24 * Per memcg lru locking
138 * memcg which the event belongs to.
140 struct mem_cgroup *memcg;
154 int (*register_event)(struct mem_cgroup *memcg,
161 void (*unregister_event)(struct mem_cgroup *memcg,
173 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
174 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
242 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
244 if (!memcg)
245 memcg = root_mem_cgroup;
246 return &memcg->vmpressure;
284 * 3) CPU1: a process from another memcg is allocating something,
329 static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
334 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
339 list_add(&objcg->list, &memcg->objcg_list);
341 list_for_each_entry(iter, &memcg->objcg_list, list)
342 WRITE_ONCE(iter->memcg, parent);
344 list_splice(&memcg->objcg_list, &parent->objcg_list);
365 * mem_cgroup_css_from_folio - css of the memcg associated with a folio
368 * If memcg is bound to the default hierarchy, css of the memcg associated
372 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
377 struct mem_cgroup *memcg = folio_memcg(folio);
379 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
380 memcg = root_mem_cgroup;
382 return &memcg->css;
386 * page_cgroup_ino - return inode number of the memcg a page is charged to
400 struct mem_cgroup *memcg;
405 memcg = folio_memcg_check(page_folio(page));
407 while (memcg && !(memcg->css.flags & CSS_ONLINE))
408 memcg = parent_mem_cgroup(memcg);
409 if (memcg)
410 ino = cgroup_ino(memcg->css.cgroup);
473 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
475 unsigned long nr_pages = page_counter_read(&memcg->memory);
476 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
485 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
492 if (soft_limit_excess(memcg))
493 lru_gen_soft_reclaim(memcg, nid);
504 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
505 mz = memcg->nodeinfo[nid];
506 excess = soft_limit_excess(memcg);
528 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
535 mz = memcg->nodeinfo[nid];
560 if (!soft_limit_excess(mz->memcg) ||
561 !css_tryget(&mz->memcg->css))
578 /* Subset of node_stat_item for memcg stats */
708 /* Subset of vm_event_item to report for memcg event stats */
797 * memcg and lruvec stats flushing
846 static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val)
855 cgroup_rstat_updated(memcg->css.cgroup, cpu);
856 statc = this_cpu_ptr(memcg->vmstats_percpu);
864 * If @memcg is already flush-able, increasing stats_updates is
874 static void do_flush_stats(struct mem_cgroup *memcg)
876 if (mem_cgroup_is_root(memcg))
879 cgroup_rstat_flush(memcg->css.cgroup);
884 * @memcg: root of the subtree to flush
891 void mem_cgroup_flush_stats(struct mem_cgroup *memcg)
896 if (!memcg)
897 memcg = root_mem_cgroup;
899 if (memcg_vmstats_needs_flush(memcg->vmstats))
900 do_flush_stats(memcg);
903 void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg)
907 mem_cgroup_flush_stats(memcg);
920 unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
928 x = READ_ONCE(memcg->vmstats->state[i]);
954 * @memcg: the memory cgroup
958 void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
969 __this_cpu_add(memcg->vmstats_percpu->state[i], val);
970 memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val));
974 static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
982 x = READ_ONCE(memcg->vmstats->state_local[i]);
995 struct mem_cgroup *memcg;
1002 memcg = pn->memcg;
1023 /* Update memcg */
1024 __this_cpu_add(memcg->vmstats_percpu->state[i], val);
1029 memcg_rstat_updated(memcg, memcg_state_val_in_pages(idx, val));
1049 /* Update memcg and lruvec */
1057 struct mem_cgroup *memcg;
1062 memcg = folio_memcg(folio);
1063 /* Untracked pages have no memcg, no lruvec. Update only the node */
1064 if (!memcg) {
1070 lruvec = mem_cgroup_lruvec(memcg, pgdat);
1079 struct mem_cgroup *memcg;
1083 memcg = mem_cgroup_from_slab_obj(p);
1086 * Untracked pages have no memcg, no lruvec. Update only the
1087 * node. If we reparent the slab objects to the root memcg,
1088 * when we free the slab object, we need to update the per-memcg
1089 * vmstats to keep it correct for the root memcg.
1091 if (!memcg) {
1094 lruvec = mem_cgroup_lruvec(memcg, pgdat);
1102 * @memcg: the memory cgroup
1106 void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
1118 __this_cpu_add(memcg->vmstats_percpu->events[i], count);
1119 memcg_rstat_updated(memcg, count);
1123 static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
1130 return READ_ONCE(memcg->vmstats->events[i]);
1133 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
1140 return READ_ONCE(memcg->vmstats->events_local[i]);
1143 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
1148 __count_memcg_events(memcg, PGPGIN, 1);
1150 __count_memcg_events(memcg, PGPGOUT, 1);
1154 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
1157 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
1162 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
1163 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
1176 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
1186 static void memcg_check_events(struct mem_cgroup *memcg, int nid)
1192 if (unlikely(mem_cgroup_event_ratelimit(memcg,
1196 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1198 mem_cgroup_threshold(memcg);
1200 mem_cgroup_update_tree(memcg, nid);
1227 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1228 * @mm: mm from which memcg should be extracted. It can be NULL.
1230 * Obtain a reference on mm->memcg and returns it if successful. If mm
1231 * is NULL, then the memcg is chosen as follows:
1232 * 1) The active memcg, if set.
1233 * 2) current->mm->memcg, if available
1234 * 3) root memcg
1239 struct mem_cgroup *memcg;
1249 * No need to css_get on root memcg as the reference
1254 memcg = active_memcg();
1255 if (unlikely(memcg)) {
1256 /* remote memcg must hold a ref */
1257 css_get(&memcg->css);
1258 return memcg;
1267 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1268 if (unlikely(!memcg))
1269 memcg = root_mem_cgroup;
1270 } while (!css_tryget(&memcg->css));
1272 return memcg;
1277 * get_mem_cgroup_from_current - Obtain a reference on current task's memcg.
1281 struct mem_cgroup *memcg;
1288 memcg = mem_cgroup_from_task(current);
1289 if (!css_tryget(&memcg->css)) {
1294 return memcg;
1300 * @prev: previously returned memcg, NULL on first invocation
1320 struct mem_cgroup *memcg = NULL;
1387 memcg = mem_cgroup_from_css(css);
1398 (void)cmpxchg(&iter->position, pos, memcg);
1403 if (!memcg)
1412 return memcg;
1445 struct mem_cgroup *memcg = dead_memcg;
1449 __invalidate_reclaim_iterators(memcg, dead_memcg);
1450 last = memcg;
1451 } while ((memcg = parent_mem_cgroup(memcg)));
1466 * @memcg: hierarchy root
1470 * This function iterates over tasks attached to @memcg or to any of its
1477 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1483 BUG_ON(mem_cgroup_is_root(memcg));
1485 for_each_mem_cgroup_tree(iter, memcg) {
1494 mem_cgroup_iter_break(memcg, iter);
1503 struct mem_cgroup *memcg;
1508 memcg = folio_memcg(folio);
1510 if (!memcg)
1513 VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio);
1627 * @memcg: the memory cgroup
1632 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1638 count = page_counter_read(&memcg->memory);
1639 limit = READ_ONCE(memcg->memory.max);
1644 count = page_counter_read(&memcg->memsw);
1645 limit = READ_ONCE(memcg->memsw.max);
1662 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1677 ret = mem_cgroup_is_descendant(from, memcg) ||
1678 mem_cgroup_is_descendant(to, memcg);
1684 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1687 if (mem_cgroup_under_move(memcg)) {
1786 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1789 return memcg_page_state(memcg, item) *
1794 struct mem_cgroup *memcg, int item)
1796 return memcg_page_state_local(memcg, item) *
1800 static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1814 mem_cgroup_flush_stats(memcg);
1819 size = memcg_page_state_output(memcg, memory_stats[i].idx);
1823 size += memcg_page_state_output(memcg,
1831 memcg_events(memcg, PGSCAN_KSWAPD) +
1832 memcg_events(memcg, PGSCAN_DIRECT) +
1833 memcg_events(memcg, PGSCAN_KHUGEPAGED));
1835 memcg_events(memcg, PGSTEAL_KSWAPD) +
1836 memcg_events(memcg, PGSTEAL_DIRECT) +
1837 memcg_events(memcg, PGSTEAL_KHUGEPAGED));
1846 memcg_events(memcg, memcg_vm_event_stat[i]));
1853 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
1855 static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1858 memcg_stat_format(memcg, s);
1860 memcg1_stat_format(memcg, s);
1867 * @memcg: The memory cgroup that went over limit
1870 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1873 void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1877 if (memcg) {
1879 pr_cont_cgroup_path(memcg->css.cgroup);
1892 * @memcg: The memory cgroup that went over limit
1894 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1903 K((u64)page_counter_read(&memcg->memory)),
1904 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
1907 K((u64)page_counter_read(&memcg->swap)),
1908 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
1911 K((u64)page_counter_read(&memcg->memsw)),
1912 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1914 K((u64)page_counter_read(&memcg->kmem)),
1915 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1919 pr_cont_cgroup_path(memcg->css.cgroup);
1922 memory_stat_format(memcg, &s);
1927 * Return the memory (and swap, if configured) limit for a memcg.
1929 unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1931 unsigned long max = READ_ONCE(memcg->memory.max);
1934 if (mem_cgroup_swappiness(memcg)) {
1936 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1941 if (mem_cgroup_swappiness(memcg))
1942 max += min(READ_ONCE(memcg->swap.max),
1948 unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1950 return page_counter_read(&memcg->memory);
1953 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1959 .memcg = memcg,
1968 if (mem_cgroup_margin(memcg) >= (1 << order))
2044 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
2050 for_each_mem_cgroup_tree(iter, memcg) {
2057 mem_cgroup_iter_break(memcg, iter);
2068 for_each_mem_cgroup_tree(iter, memcg) {
2070 mem_cgroup_iter_break(memcg, iter);
2083 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2089 for_each_mem_cgroup_tree(iter, memcg)
2094 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2099 for_each_mem_cgroup_tree(iter, memcg)
2104 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2109 * Be careful about under_oom underflows because a child memcg
2113 for_each_mem_cgroup_tree(iter, memcg)
2122 struct mem_cgroup *memcg;
2134 oom_wait_memcg = oom_wait_info->memcg;
2142 static void memcg_oom_recover(struct mem_cgroup *memcg)
2152 if (memcg && memcg->under_oom)
2153 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2160 static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
2167 memcg_memory_event(memcg, MEMCG_OOM);
2187 if (READ_ONCE(memcg->oom_kill_disable)) {
2189 css_get(&memcg->css);
2190 current->memcg_in_oom = memcg;
2195 mem_cgroup_mark_under_oom(memcg);
2197 locked = mem_cgroup_oom_trylock(memcg);
2200 mem_cgroup_oom_notify(memcg);
2202 mem_cgroup_unmark_under_oom(memcg);
2203 ret = mem_cgroup_out_of_memory(memcg, mask, order);
2206 mem_cgroup_oom_unlock(memcg);
2212 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2215 * This has to be called at the end of a page fault if the memcg OOM
2225 * Returns %true if an ongoing memcg OOM situation was detected and
2230 struct mem_cgroup *memcg = current->memcg_in_oom;
2235 if (!memcg)
2241 owait.memcg = memcg;
2248 mem_cgroup_mark_under_oom(memcg);
2250 locked = mem_cgroup_oom_trylock(memcg);
2253 mem_cgroup_oom_notify(memcg);
2256 mem_cgroup_unmark_under_oom(memcg);
2260 mem_cgroup_oom_unlock(memcg);
2263 css_put(&memcg->css);
2270 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2275 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2281 struct mem_cgroup *memcg;
2291 memcg = mem_cgroup_from_task(victim);
2292 if (mem_cgroup_is_root(memcg))
2300 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2308 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2309 if (READ_ONCE(memcg->oom_group))
2310 oom_group = memcg;
2312 if (memcg == oom_domain)
2324 void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2327 pr_cont_cgroup_path(memcg->css.cgroup);
2332 * folio_memcg_lock - Bind a folio to its memcg.
2338 * It ensures lifetime of the bound memcg. The caller is responsible
2343 struct mem_cgroup *memcg;
2348 * path can get away without acquiring the memcg->move_lock
2356 memcg = folio_memcg(folio);
2357 if (unlikely(!memcg))
2362 might_lock(&memcg->move_lock);
2366 if (atomic_read(&memcg->moving_account) <= 0)
2369 spin_lock_irqsave(&memcg->move_lock, flags);
2370 if (memcg != folio_memcg(folio)) {
2371 spin_unlock_irqrestore(&memcg->move_lock, flags);
2381 memcg->move_lock_task = current;
2382 memcg->move_lock_flags = flags;
2385 static void __folio_memcg_unlock(struct mem_cgroup *memcg)
2387 if (memcg && memcg->move_lock_task == current) {
2388 unsigned long flags = memcg->move_lock_flags;
2390 memcg->move_lock_task = NULL;
2391 memcg->move_lock_flags = 0;
2393 spin_unlock_irqrestore(&memcg->move_lock, flags);
2400 * folio_memcg_unlock - Release the binding between a folio and its memcg.
2404 * not change the accounting of this folio to its memcg, but it does
2438 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages);
2450 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
2457 * @memcg: memcg to consume from.
2460 * The charges will only happen if @memcg matches the current cpu's memcg
2466 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2480 if (memcg == READ_ONCE(stock->cached) && stock_pages >= nr_pages) {
2539 static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2545 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */
2547 css_get(&memcg->css);
2548 WRITE_ONCE(stock->cached, memcg);
2557 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2562 __refill_stock(memcg, nr_pages);
2587 struct mem_cgroup *memcg;
2591 memcg = READ_ONCE(stock->cached);
2592 if (memcg && READ_ONCE(stock->nr_pages) &&
2593 mem_cgroup_is_descendant(memcg, root_memcg))
2621 static unsigned long reclaim_high(struct mem_cgroup *memcg,
2630 if (page_counter_read(&memcg->memory) <=
2631 READ_ONCE(memcg->memory.high))
2634 memcg_memory_event(memcg, MEMCG_HIGH);
2637 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2641 } while ((memcg = parent_mem_cgroup(memcg)) &&
2642 !mem_cgroup_is_root(memcg));
2649 struct mem_cgroup *memcg;
2651 memcg = container_of(work, struct mem_cgroup, high_work);
2652 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2726 static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2731 overage = calculate_overage(page_counter_read(&memcg->memory),
2732 READ_ONCE(memcg->memory.high));
2734 } while ((memcg = parent_mem_cgroup(memcg)) &&
2735 !mem_cgroup_is_root(memcg));
2740 static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2745 overage = calculate_overage(page_counter_read(&memcg->swap),
2746 READ_ONCE(memcg->swap.high));
2748 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2750 } while ((memcg = parent_mem_cgroup(memcg)) &&
2751 !mem_cgroup_is_root(memcg));
2760 static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2773 * memcg in question makes it clear that it has no intention of stopping
2804 struct mem_cgroup *memcg;
2810 memcg = get_mem_cgroup_from_mm(current->mm);
2834 nr_reclaimed = reclaim_high(memcg,
2842 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2843 mem_find_max_overage(memcg));
2845 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2846 swap_find_max_overage(memcg));
2856 * Don't sleep if the amount of jiffies this memcg owes us is so low
2887 css_put(&memcg->css);
2890 static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2905 if (consume_stock(memcg, nr_pages))
2909 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2910 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2913 page_counter_uncharge(&memcg->memsw, batch);
2988 * keep retrying as long as the memcg oom killer is able to make
3020 page_counter_charge(&memcg->memory, nr_pages);
3022 page_counter_charge(&memcg->memsw, nr_pages);
3028 refill_stock(memcg, batch - nr_pages);
3034 * GFP_KERNEL can consistently be used during reclaim. @memcg is
3042 mem_high = page_counter_read(&memcg->memory) >
3043 READ_ONCE(memcg->memory.high);
3044 swap_high = page_counter_read(&memcg->swap) >
3045 READ_ONCE(memcg->swap.high);
3050 schedule_work(&memcg->high_work);
3070 } while ((memcg = parent_mem_cgroup(memcg)));
3086 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
3089 if (mem_cgroup_is_root(memcg))
3092 return try_charge_memcg(memcg, gfp_mask, nr_pages);
3097 * @memcg: memcg previously charged.
3100 void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
3102 if (mem_cgroup_is_root(memcg))
3105 page_counter_uncharge(&memcg->memory, nr_pages);
3107 page_counter_uncharge(&memcg->memsw, nr_pages);
3110 static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
3114 * Any of the following ensures page's memcg stability:
3122 folio->memcg_data = (unsigned long)memcg;
3128 * @memcg: memcg previously charged.
3130 void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
3132 css_get(&memcg->css);
3133 commit_charge(folio, memcg);
3136 mem_cgroup_charge_statistics(memcg, folio_nr_pages(folio));
3137 memcg_check_events(memcg, folio_nid(folio));
3147 struct mem_cgroup *memcg;
3153 memcg = obj_cgroup_memcg(objcg);
3154 lruvec = mem_cgroup_lruvec(memcg, pgdat);
3205 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
3230 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
3241 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
3245 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
3246 objcg = rcu_dereference(memcg->objcg);
3256 struct mem_cgroup *memcg;
3285 * asynchronously moved to another memcg and the previous
3286 * memcg can be offlined. So let's get the memcg pointer
3291 memcg = mem_cgroup_from_task(current);
3292 objcg = __get_obj_cgroup_from_memcg(memcg);
3307 struct mem_cgroup *memcg;
3311 memcg = current->active_memcg;
3312 if (unlikely(memcg))
3325 memcg = this_cpu_read(int_active_memcg);
3326 if (unlikely(memcg))
3333 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
3340 objcg = rcu_dereference_check(memcg->objcg, 1);
3359 struct mem_cgroup *memcg;
3362 memcg = __folio_memcg(folio);
3363 if (memcg)
3364 objcg = __get_obj_cgroup_from_memcg(memcg);
3372 static void memcg_account_kmem(struct mem_cgroup *memcg, int nr_pages)
3374 mod_memcg_state(memcg, MEMCG_KMEM, nr_pages);
3377 page_counter_charge(&memcg->kmem, nr_pages);
3379 page_counter_uncharge(&memcg->kmem, -nr_pages);
3392 struct mem_cgroup *memcg;
3394 memcg = get_mem_cgroup_from_objcg(objcg);
3396 memcg_account_kmem(memcg, -nr_pages);
3397 refill_stock(memcg, nr_pages);
3399 css_put(&memcg->css);
3413 struct mem_cgroup *memcg;
3416 memcg = get_mem_cgroup_from_objcg(objcg);
3418 ret = try_charge_memcg(memcg, gfp, nr_pages);
3422 memcg_account_kmem(memcg, nr_pages);
3424 css_put(&memcg->css);
3571 struct mem_cgroup *memcg;
3573 memcg = get_mem_cgroup_from_objcg(old);
3575 memcg_account_kmem(memcg, -nr_pages);
3576 __refill_stock(memcg, nr_pages);
3578 css_put(&memcg->css);
3582 * The leftover is flushed to the centralized per-memcg value.
3626 struct mem_cgroup *memcg;
3629 memcg = obj_cgroup_memcg(objcg);
3630 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3684 * performance bottleneck if all tasks of the same memcg are
3758 struct mem_cgroup *memcg;
3760 memcg = get_mem_cgroup_from_objcg(objcg);
3761 ret = memcg_list_lru_alloc(memcg, lru, flags);
3762 css_put(&memcg->css);
3817 struct mem_cgroup *memcg = folio_memcg(folio);
3822 if (mem_cgroup_disabled() || !memcg)
3831 css_get_many(&memcg->css, old_nr / new_nr - 1);
3874 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3881 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3894 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
3895 max <= memcg->memsw.max;
3910 drain_all_stock(memcg);
3915 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
3923 memcg_oom_recover(memcg);
3968 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3981 excess = soft_limit_excess(mz->memcg);
3993 css_put(&mz->memcg->css);
4006 css_put(&next_mz->memcg->css);
4011 * Reclaims as many pages from the given memcg as possible.
4013 * Caller is responsible for holding css reference for memcg.
4015 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4022 drain_all_stock(memcg);
4025 while (nr_retries && page_counter_read(&memcg->memory)) {
4029 if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
4041 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4043 if (mem_cgroup_is_root(memcg))
4045 return mem_cgroup_force_empty(memcg) ?: nbytes;
4067 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
4071 if (mem_cgroup_is_root(memcg)) {
4082 val = page_counter_read(&memcg->memory);
4084 val = page_counter_read(&memcg->memsw);
4100 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4105 counter = &memcg->memory;
4108 counter = &memcg->memsw;
4111 counter = &memcg->kmem;
4114 counter = &memcg->tcpmem;
4122 if (counter == &memcg->memory)
4123 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
4124 if (counter == &memcg->memsw)
4125 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
4134 return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE;
4151 static int memcg_online_kmem(struct mem_cgroup *memcg)
4158 if (unlikely(mem_cgroup_is_root(memcg)))
4165 objcg->memcg = memcg;
4166 rcu_assign_pointer(memcg->objcg, objcg);
4168 memcg->orig_objcg = objcg;
4172 memcg->kmemcg_id = memcg->id.id;
4177 static void memcg_offline_kmem(struct mem_cgroup *memcg)
4184 if (unlikely(mem_cgroup_is_root(memcg)))
4187 parent = parent_mem_cgroup(memcg);
4191 memcg_reparent_objcgs(memcg, parent);
4199 memcg_reparent_list_lrus(memcg, parent);
4202 static int memcg_online_kmem(struct mem_cgroup *memcg)
4206 static void memcg_offline_kmem(struct mem_cgroup *memcg)
4211 static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
4217 ret = page_counter_set_max(&memcg->tcpmem, max);
4221 if (!memcg->tcpmem_active) {
4227 * belonging to this memcg until that flag is up.
4239 memcg->tcpmem_active = true;
4253 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4264 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
4270 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
4273 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
4283 ret = memcg_update_tcp_max(memcg, nr_pages);
4291 WRITE_ONCE(memcg->soft_limit, nr_pages);
4302 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4307 counter = &memcg->memory;
4310 counter = &memcg->memsw;
4313 counter = &memcg->kmem;
4316 counter = &memcg->tcpmem;
4346 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4361 memcg->move_charge_at_immigrate = val;
4378 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
4381 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
4398 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
4409 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
4411 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
4431 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4433 mem_cgroup_flush_stats(memcg);
4437 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4441 mem_cgroup_node_nr_lru_pages(memcg, nid,
4449 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
4453 mem_cgroup_node_nr_lru_pages(memcg, nid,
4506 static void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
4514 mem_cgroup_flush_stats(memcg);
4519 nr = memcg_page_state_local_output(memcg, memcg1_stats[i]);
4525 memcg_events_local(memcg, memcg1_events[i]));
4529 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
4534 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
4546 nr = memcg_page_state_output(memcg, memcg1_stats[i]);
4554 (u64)memcg_events(memcg, memcg1_events[i]));
4558 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4569 mz = memcg->nodeinfo[pgdat->node_id];
4583 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4585 return mem_cgroup_swappiness(memcg);
4591 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4596 if (!mem_cgroup_is_root(memcg))
4597 WRITE_ONCE(memcg->swappiness, val);
4604 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4612 t = rcu_dereference(memcg->thresholds.primary);
4614 t = rcu_dereference(memcg->memsw_thresholds.primary);
4619 usage = mem_cgroup_usage(memcg, swap);
4655 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4657 while (memcg) {
4658 __mem_cgroup_threshold(memcg, false);
4660 __mem_cgroup_threshold(memcg, true);
4662 memcg = parent_mem_cgroup(memcg);
4680 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
4686 list_for_each_entry(ev, &memcg->oom_notify, list)
4693 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
4697 for_each_mem_cgroup_tree(iter, memcg)
4701 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4714 mutex_lock(&memcg->thresholds_lock);
4717 thresholds = &memcg->thresholds;
4718 usage = mem_cgroup_usage(memcg, false);
4720 thresholds = &memcg->memsw_thresholds;
4721 usage = mem_cgroup_usage(memcg, true);
4727 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4776 mutex_unlock(&memcg->thresholds_lock);
4781 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
4784 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
4787 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
4790 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
4793 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4801 mutex_lock(&memcg->thresholds_lock);
4804 thresholds = &memcg->thresholds;
4805 usage = mem_cgroup_usage(memcg, false);
4807 thresholds = &memcg->memsw_thresholds;
4808 usage = mem_cgroup_usage(memcg, true);
4816 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4875 mutex_unlock(&memcg->thresholds_lock);
4878 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4881 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
4884 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
4887 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
4890 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
4902 list_add(&event->list, &memcg->oom_notify);
4905 if (memcg->under_oom)
4912 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
4919 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
4931 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4933 seq_printf(sf, "oom_kill_disable %d\n", READ_ONCE(memcg->oom_kill_disable));
4934 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
4936 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4943 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4946 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
4949 WRITE_ONCE(memcg->oom_kill_disable, val);
4951 memcg_oom_recover(memcg);
4960 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4962 return wb_domain_init(&memcg->cgwb_domain, gfp);
4965 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4967 wb_domain_exit(&memcg->cgwb_domain);
4970 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4972 wb_domain_size_changed(&memcg->cgwb_domain);
4977 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4979 if (!memcg->css.parent)
4982 return &memcg->cgwb_domain;
4986 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4989 * @pheadroom: out parameter for number of allocatable pages according to memcg
4994 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4997 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
5007 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
5010 mem_cgroup_flush_stats_ratelimited(memcg);
5012 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
5013 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
5014 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
5015 memcg_page_state(memcg, NR_ACTIVE_FILE);
5018 while ((parent = parent_mem_cgroup(memcg))) {
5019 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
5020 READ_ONCE(memcg->memory.high));
5021 unsigned long used = page_counter_read(&memcg->memory);
5024 memcg = parent;
5031 * There's an inherent mismatch between memcg and writeback. The former
5053 * granularities between memcg and writeback in either direction. However,
5059 * page - a page whose memcg and writeback ownerships don't match - is
5061 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
5062 * decides that the memcg needs to sleep due to high dirty ratio, it calls
5075 struct mem_cgroup *memcg = folio_memcg(folio);
5090 frn = &memcg->cgwb_frn[i];
5117 frn = &memcg->cgwb_frn[oldest];
5127 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
5133 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
5154 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
5159 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
5163 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
5191 struct mem_cgroup *memcg = event->memcg;
5195 event->unregister_event(memcg, event->eventfd);
5202 css_put(&memcg->css);
5215 struct mem_cgroup *memcg = event->memcg;
5228 spin_lock(&memcg->event_list_lock);
5237 spin_unlock(&memcg->event_list_lock);
5265 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5295 event->memcg = memcg;
5377 ret = event->register_event(memcg, event->eventfd, buf);
5383 spin_lock_irq(&memcg->event_list_lock);
5384 list_add(&event->list, &memcg->event_list);
5385 spin_unlock_irq(&memcg->event_list_lock);
5550 * Swap-out records and page cache shadow entries need to store memcg
5562 * Maintain a private 16-bit ID space for memcg, and allow the ID to
5574 static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5576 if (memcg->id.id > 0) {
5577 idr_remove(&mem_cgroup_idr, memcg->id.id);
5578 memcg->id.id = 0;
5582 static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5585 refcount_add(n, &memcg->id.ref);
5588 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
5590 if (refcount_sub_and_test(n, &memcg->id.ref)) {
5591 mem_cgroup_id_remove(memcg);
5594 css_put(&memcg->css);
5598 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5600 mem_cgroup_id_put_many(memcg, 1);
5604 * mem_cgroup_from_id - look up a memcg from a memcg id
5605 * @id: the memcg id to look up
5620 struct mem_cgroup *memcg;
5628 memcg = container_of(css, struct mem_cgroup, css);
5630 memcg = ERR_PTR(-ENOENT);
5634 return memcg;
5638 static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5657 pn->memcg = memcg;
5659 memcg->nodeinfo[node] = pn;
5667 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
5669 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5679 static void __mem_cgroup_free(struct mem_cgroup *memcg)
5683 obj_cgroup_put(memcg->orig_objcg);
5686 free_mem_cgroup_per_node_info(memcg, node);
5687 kfree(memcg->vmstats);
5688 free_percpu(memcg->vmstats_percpu);
5689 kfree(memcg);
5692 static void mem_cgroup_free(struct mem_cgroup *memcg)
5694 lru_gen_exit_memcg(memcg);
5695 memcg_wb_domain_exit(memcg);
5696 __mem_cgroup_free(memcg);
5702 struct mem_cgroup *memcg;
5707 memcg = kzalloc(struct_size(memcg, nodeinfo, nr_node_ids), GFP_KERNEL);
5708 if (!memcg)
5711 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5713 if (memcg->id.id < 0) {
5714 error = memcg->id.id;
5718 memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats),
5720 if (!memcg->vmstats)
5723 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5725 if (!memcg->vmstats_percpu)
5731 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5733 statc->vmstats = memcg->vmstats;
5737 if (!alloc_mem_cgroup_per_node_info(memcg, node))
5740 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5743 INIT_WORK(&memcg->high_work, high_work_func);
5744 INIT_LIST_HEAD(&memcg->oom_notify);
5745 mutex_init(&memcg->thresholds_lock);
5746 spin_lock_init(&memcg->move_lock);
5747 vmpressure_init(&memcg->vmpressure);
5748 INIT_LIST_HEAD(&memcg->event_list);
5749 spin_lock_init(&memcg->event_list_lock);
5750 memcg->socket_pressure = jiffies;
5752 memcg->kmemcg_id = -1;
5753 INIT_LIST_HEAD(&memcg->objcg_list);
5756 INIT_LIST_HEAD(&memcg->cgwb_list);
5758 memcg->cgwb_frn[i].done =
5762 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5763 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5764 memcg->deferred_split_queue.split_queue_len = 0;
5766 lru_gen_init_memcg(memcg);
5767 return memcg;
5769 mem_cgroup_id_remove(memcg);
5770 __mem_cgroup_free(memcg);
5778 struct mem_cgroup *memcg, *old_memcg;
5781 memcg = mem_cgroup_alloc(parent);
5783 if (IS_ERR(memcg))
5784 return ERR_CAST(memcg);
5786 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5787 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
5789 memcg->zswap_max = PAGE_COUNTER_MAX;
5790 WRITE_ONCE(memcg->zswap_writeback,
5793 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5795 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
5796 WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
5798 page_counter_init(&memcg->memory, &parent->memory);
5799 page_counter_init(&memcg->swap, &parent->swap);
5800 page_counter_init(&memcg->kmem, &parent->kmem);
5801 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5805 page_counter_init(&memcg->memory, NULL);
5806 page_counter_init(&memcg->swap, NULL);
5807 page_counter_init(&memcg->kmem, NULL);
5808 page_counter_init(&memcg->tcpmem, NULL);
5810 root_mem_cgroup = memcg;
5811 return &memcg->css;
5822 return &memcg->css;
5827 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5829 if (memcg_online_kmem(memcg))
5833 * A memcg must be visible for expand_shrinker_info()
5837 if (alloc_shrinker_info(memcg))
5840 if (unlikely(mem_cgroup_is_root(memcg)) && !mem_cgroup_disabled())
5843 lru_gen_online_memcg(memcg);
5845 /* Online state pins memcg ID, memcg ID pins CSS */
5846 refcount_set(&memcg->id.ref, 1);
5859 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5863 memcg_offline_kmem(memcg);
5865 mem_cgroup_id_remove(memcg);
5871 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5879 spin_lock_irq(&memcg->event_list_lock);
5880 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5884 spin_unlock_irq(&memcg->event_list_lock);
5886 page_counter_set_min(&memcg->memory, 0);
5887 page_counter_set_low(&memcg->memory, 0);
5889 zswap_memcg_offline_cleanup(memcg);
5891 memcg_offline_kmem(memcg);
5892 reparent_shrinker_deferred(memcg);
5893 wb_memcg_offline(memcg);
5894 lru_gen_offline_memcg(memcg);
5896 drain_all_stock(memcg);
5898 mem_cgroup_id_put(memcg);
5903 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5905 invalidate_reclaim_iterators(memcg);
5906 lru_gen_release_memcg(memcg);
5911 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5916 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5921 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
5929 vmpressure_cleanup(&memcg->vmpressure);
5930 cancel_work_sync(&memcg->high_work);
5931 mem_cgroup_remove_from_trees(memcg);
5932 free_shrinker_info(memcg);
5933 mem_cgroup_free(memcg);
5942 * but the memcg is pinned through dependency. The memcg should stop
5951 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5953 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5954 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
5955 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5956 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
5957 page_counter_set_min(&memcg->memory, 0);
5958 page_counter_set_low(&memcg->memory, 0);
5959 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
5960 WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
5961 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
5962 memcg_wb_domain_size_changed(memcg);
5967 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5968 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5973 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
5981 delta = memcg->vmstats->state_pending[i];
5983 memcg->vmstats->state_pending[i] = 0;
5996 memcg->vmstats->state_local[i] += delta_cpu;
5999 memcg->vmstats->state[i] += delta;
6006 delta = memcg->vmstats->events_pending[i];
6008 memcg->vmstats->events_pending[i] = 0;
6019 memcg->vmstats->events_local[i] += delta_cpu;
6022 memcg->vmstats->events[i] += delta;
6029 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
6064 if (atomic64_read(&memcg->vmstats->stats_updates))
6065 atomic64_set(&memcg->vmstats->stats_updates, 0);
6270 * All state has been migrated, let's switch to the new memcg.
6272 * It is safe to change page's memcg here because the page
6277 * Note that folio_memcg_lock is a memcg lock, not a page lock,
6279 * new memcg that isn't locked, the above state can change
6572 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
6593 memcg = mem_cgroup_from_css(css);
6603 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
6609 VM_BUG_ON(from == memcg);
6625 mc.to = memcg;
6712 * memcg. There should be somebody mapping the head.
6770 * Signal folio_memcg_lock() to take the memcg's move_lock
6771 * while we're moving its pages to another memcg. Then wait
6907 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6909 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
6915 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6917 return (u64)memcg->memory.watermark * PAGE_SIZE;
6929 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6938 page_counter_set_min(&memcg->memory, min);
6952 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6961 page_counter_set_low(&memcg->memory, low);
6975 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6986 page_counter_set_high(&memcg->memory, high);
6989 unsigned long nr_pages = page_counter_read(&memcg->memory);
6999 drain_all_stock(memcg);
7004 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
7011 memcg_wb_domain_size_changed(memcg);
7024 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7035 xchg(&memcg->memory.max, max);
7038 unsigned long nr_pages = page_counter_read(&memcg->memory);
7047 drain_all_stock(memcg);
7053 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
7059 memcg_memory_event(memcg, MEMCG_OOM);
7060 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
7064 memcg_wb_domain_size_changed(memcg);
7086 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7088 __memory_events_show(m, memcg->memory_events);
7094 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7096 __memory_events_show(m, memcg->memory_events_local);
7102 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7109 memory_stat_format(memcg, &s);
7126 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7128 mem_cgroup_flush_stats(memcg);
7141 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
7155 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7157 seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group));
7165 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7179 WRITE_ONCE(memcg->oom_group, oom_group);
7187 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7215 reclaimed = try_to_free_mem_cgroup_pages(memcg,
7440 * @memcg: the memory cgroup to check
7446 struct mem_cgroup *memcg)
7464 if (memcg == root)
7467 usage = page_counter_read(&memcg->memory);
7471 parent = parent_mem_cgroup(memcg);
7474 memcg->memory.emin = READ_ONCE(memcg->memory.min);
7475 memcg->memory.elow = READ_ONCE(memcg->memory.low);
7481 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
7482 READ_ONCE(memcg->memory.min),
7486 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
7487 READ_ONCE(memcg->memory.low),
7492 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
7497 ret = try_charge(memcg, gfp, folio_nr_pages(folio));
7501 mem_cgroup_commit_charge(folio, memcg);
7508 struct mem_cgroup *memcg;
7511 memcg = get_mem_cgroup_from_mm(mm);
7512 ret = charge_memcg(folio, memcg, gfp);
7513 css_put(&memcg->css);
7519 * mem_cgroup_hugetlb_try_charge - try to charge the memcg for a hugetlb folio
7520 * @memcg: memcg to charge.
7525 * the memcg has the capacity for it. It does not commit the charge yet,
7535 int mem_cgroup_hugetlb_try_charge(struct mem_cgroup *memcg, gfp_t gfp,
7539 * If hugetlb memcg charging is not enabled, do not fail hugetlb allocation,
7542 if (mem_cgroup_disabled() || !memcg ||
7547 if (try_charge(memcg, gfp, nr_pages))
7568 struct mem_cgroup *memcg;
7577 memcg = mem_cgroup_from_id(id);
7578 if (!memcg || !css_tryget_online(&memcg->css))
7579 memcg = get_mem_cgroup_from_mm(mm);
7582 ret = charge_memcg(folio, memcg, gfp);
7584 css_put(&memcg->css);
7622 struct mem_cgroup *memcg;
7639 page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
7641 page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
7643 memcg_account_kmem(ug->memcg, -ug->nr_kmem);
7644 memcg_oom_recover(ug->memcg);
7648 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
7649 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_memory);
7650 memcg_check_events(ug->memcg, ug->nid);
7654 css_put(&ug->memcg->css);
7660 struct mem_cgroup *memcg;
7670 * folio memcg or objcg at this point, we have fully
7677 * kmem pages do not hold memcg references anymore.
7679 memcg = get_mem_cgroup_from_objcg(objcg);
7681 memcg = __folio_memcg(folio);
7684 if (!memcg)
7687 if (ug->memcg != memcg) {
7688 if (ug->memcg) {
7692 ug->memcg = memcg;
7696 css_get(&memcg->css);
7709 if (!mem_cgroup_is_root(memcg))
7716 css_put(&memcg->css);
7740 if (ug.memcg)
7757 struct mem_cgroup *memcg;
7773 memcg = folio_memcg(old);
7774 VM_WARN_ON_ONCE_FOLIO(!memcg, old);
7775 if (!memcg)
7779 if (!mem_cgroup_is_root(memcg)) {
7780 page_counter_charge(&memcg->memory, nr_pages);
7782 page_counter_charge(&memcg->memsw, nr_pages);
7785 css_get(&memcg->css);
7786 commit_charge(new, memcg);
7789 mem_cgroup_charge_statistics(memcg, nr_pages);
7790 memcg_check_events(memcg, folio_nid(new));
7795 * mem_cgroup_migrate - Transfer the memcg data from the old to the new folio.
7799 * Transfer the memcg data from the old folio to the new folio for migration.
7807 struct mem_cgroup *memcg;
7817 memcg = folio_memcg(old);
7819 * Note that it is normal to see !memcg for a hugetlb folio.
7823 VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(old) && !memcg, old);
7824 if (!memcg)
7828 commit_charge(new, memcg);
7832 * split queue in destroy_large_folio() after the memcg of the old folio
7848 struct mem_cgroup *memcg;
7853 /* Do not associate the sock with unrelated interrupted task's memcg. */
7858 memcg = mem_cgroup_from_task(current);
7859 if (mem_cgroup_is_root(memcg))
7861 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
7863 if (css_tryget(&memcg->css))
7864 sk->sk_memcg = memcg;
7877 * @memcg: memcg to charge
7881 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
7882 * @memcg's configured limit, %false if it doesn't.
7884 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
7890 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
7891 memcg->tcpmem_pressure = 0;
7894 memcg->tcpmem_pressure = 1;
7896 page_counter_charge(&memcg->tcpmem, nr_pages);
7902 if (try_charge(memcg, gfp_mask, nr_pages) == 0) {
7903 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
7912 * @memcg: memcg to uncharge
7915 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7918 page_counter_uncharge(&memcg->tcpmem, nr_pages);
7922 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
7924 refill_stock(memcg, nr_pages);
7959 * used for per-memcg-per-cpu caching of per-node statistics. In order
7988 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7990 while (!refcount_inc_not_zero(&memcg->id.ref)) {
7995 if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) {
7999 memcg = parent_mem_cgroup(memcg);
8000 if (!memcg)
8001 memcg = root_mem_cgroup;
8003 return memcg;
8015 struct mem_cgroup *memcg, *swap_memcg;
8028 memcg = folio_memcg(folio);
8030 VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
8031 if (!memcg)
8035 * In case the memcg owning these pages has been offlined and doesn't
8039 swap_memcg = mem_cgroup_id_get_online(memcg);
8051 if (!mem_cgroup_is_root(memcg))
8052 page_counter_uncharge(&memcg->memory, nr_entries);
8054 if (memcg != swap_memcg) {
8057 page_counter_uncharge(&memcg->memsw, nr_entries);
8067 mem_cgroup_charge_statistics(memcg, -nr_entries);
8069 memcg_check_events(memcg, folio_nid(folio));
8071 css_put(&memcg->css);
8079 * Try to charge @folio's memcg for the swap space at @entry.
8087 struct mem_cgroup *memcg;
8093 memcg = folio_memcg(folio);
8095 VM_WARN_ON_ONCE_FOLIO(!memcg, folio);
8096 if (!memcg)
8100 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
8104 memcg = mem_cgroup_id_get_online(memcg);
8106 if (!mem_cgroup_is_root(memcg) &&
8107 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
8108 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
8109 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
8110 mem_cgroup_id_put(memcg);
8116 mem_cgroup_id_get_many(memcg, nr_pages - 1);
8117 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
8119 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
8131 struct mem_cgroup *memcg;
8136 memcg = mem_cgroup_from_id(id);
8137 if (memcg) {
8138 if (!mem_cgroup_is_root(memcg)) {
8140 page_counter_uncharge(&memcg->memsw, nr_pages);
8142 page_counter_uncharge(&memcg->swap, nr_pages);
8144 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
8145 mem_cgroup_id_put_many(memcg, nr_pages);
8150 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
8156 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
8158 READ_ONCE(memcg->swap.max) -
8159 page_counter_read(&memcg->swap));
8165 struct mem_cgroup *memcg;
8174 memcg = folio_memcg(folio);
8175 if (!memcg)
8178 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
8179 unsigned long usage = page_counter_read(&memcg->swap);
8181 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
8182 usage * 2 >= READ_ONCE(memcg->swap.max))
8205 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
8207 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
8213 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
8215 return (u64)memcg->swap.watermark * PAGE_SIZE;
8227 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
8236 page_counter_set_high(&memcg->swap, high);
8250 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
8259 xchg(&memcg->swap.max, max);
8266 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
8269 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
8271 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
8273 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
8352 struct mem_cgroup *memcg, *original_memcg;
8359 for (memcg = original_memcg; !mem_cgroup_is_root(memcg);
8360 memcg = parent_mem_cgroup(memcg)) {
8361 unsigned long max = READ_ONCE(memcg->zswap_max);
8375 do_flush_stats(memcg);
8376 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE;
8396 struct mem_cgroup *memcg;
8408 memcg = obj_cgroup_memcg(objcg);
8409 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
8410 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
8423 struct mem_cgroup *memcg;
8431 memcg = obj_cgroup_memcg(objcg);
8432 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size);
8433 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1);
8437 bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
8440 return !is_zswap_enabled() || !memcg || READ_ONCE(memcg->zswap_writeback);
8446 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
8448 mem_cgroup_flush_stats(memcg);
8449 return memcg_page_state(memcg, MEMCG_ZSWAP_B);
8461 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
8470 xchg(&memcg->zswap_max, max);
8477 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
8479 seq_printf(m, "%d\n", READ_ONCE(memcg->zswap_writeback));
8486 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
8496 WRITE_ONCE(memcg->zswap_writeback, zswap_writeback);