include/linux/mm_inline.h

38 static __always_inline void __update_lru_size(struct lruvec *lruvec,
42 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
44 	lockdep_assert_held(&lruvec->lru_lock);
47 	__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
52 static __always_inline void update_lru_size(struct lruvec *lruvec,
56 	__update_lru_size(lruvec, lru, zid, nr_pages);
58 	mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
165 static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
167 	unsigned long max_seq = lruvec->lrugen.max_seq;
175 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
182 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
197 		if (lru_gen_is_active(lruvec, new_gen))
199 		__update_lru_size(lruvec, lru, zone, delta);
205 		if (lru_gen_is_active(lruvec, old_gen))
207 		__update_lru_size(lruvec, lru, zone, -delta);
212 	if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
213 		__update_lru_size(lruvec, lru, zone, -delta);
214 		__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
218 	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
221 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
228 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
262 	lru_gen_update_size(lruvec, folio, -1, gen);
272 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
284 	flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
288 	lru_gen_update_size(lruvec, folio, gen, -1);
306 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
311 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
319 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
323 	if (lru_gen_add_folio(lruvec, folio, false))
326 	update_lru_size(lruvec, lru, folio_zonenum(folio),
329 		list_add(&folio->lru, &lruvec->lists[lru]);
333 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
337 	if (lru_gen_add_folio(lruvec, folio, true))
340 	update_lru_size(lruvec, lru, folio_zonenum(folio),
343 	list_add_tail(&folio->lru, &lruvec->lists[lru]);
347 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
351 	if (lru_gen_del_folio(lruvec, folio, false))
356 	update_lru_size(lruvec, lru, folio_zonenum(folio),