linux-master/mm/vmscan.c

2589 	unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
2593 		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]),	\
2594 		READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]),	\
2642 	return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
3051 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3052 	int hist = lru_hist_from_seq(lrugen->min_seq[type]);
3054 	pos->refaulted = lrugen->avg_refaulted[type][tier] +
3055 			 atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3056 	pos->total = lrugen->avg_total[type][tier] +
3057 		     atomic_long_read(&lrugen->evicted[hist][type][tier]);
3059 		pos->total += lrugen->protected[hist][type][tier - 1];
3066 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3068 	unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
3081 			sum = lrugen->avg_refaulted[type][tier] +
3082 			      atomic_long_read(&lrugen->refaulted[hist][type][tier]);
3083 			WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
3085 			sum = lrugen->avg_total[type][tier] +
3086 			      atomic_long_read(&lrugen->evicted[hist][type][tier]);
3088 				sum += lrugen->protected[hist][type][tier - 1];
3089 			WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
3093 			atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
3094 			atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
3096 				WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
3143 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3144 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3189 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3201 		WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
3202 			   lrugen->nr_pages[gen][type][zone] + delta);
3702 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3703 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
3710 		struct list_head *head = &lrugen->folios[old_gen][type][zone];
3721 			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
3729 	WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
3738 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3745 		while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
3749 				if (!list_empty(&lrugen->folios[gen][type][zone]))
3762 		min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
3766 		if (min_seq[type] == lrugen->min_seq[type])
3770 		WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
3783 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3785 	if (seq < READ_ONCE(lrugen->max_seq))
3792 	success = seq == lrugen->max_seq;
3816 	prev = lru_gen_from_seq(lrugen->max_seq - 1);
3817 	next = lru_gen_from_seq(lrugen->max_seq + 1);
3822 			long delta = lrugen->nr_pages[prev][type][zone] -
3823 				     lrugen->nr_pages[next][type][zone];
3836 	WRITE_ONCE(lrugen->timestamps[next], jiffies);
3838 	smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
3851 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3854 	VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq));
3908 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
3920 				total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
3938 	birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
4129 	VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
4132 	new = old = lruvec->lrugen.gen;
4146 	WRITE_ONCE(lruvec->lrugen.seg, seg);
4147 	WRITE_ONCE(lruvec->lrugen.gen, new);
4149 	hlist_nulls_del_rcu(&lruvec->lrugen.list);
4152 		hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4154 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
4179 		VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
4183 		lruvec->lrugen.gen = gen;
4185 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
4214 		if (hlist_nulls_unhashed(&lruvec->lrugen.list))
4217 		gen = lruvec->lrugen.gen;
4219 		hlist_nulls_del_init_rcu(&lruvec->lrugen.list);
4234 	if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD)
4254 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4278 	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
4279 		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4285 		int hist = lru_hist_from_seq(lrugen->min_seq[type]);
4288 		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4290 		WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
4291 			   lrugen->protected[hist][type][tier - 1] + delta);
4298 		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
4306 		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
4358 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4366 	gen = lru_gen_from_seq(lrugen->min_seq[type]);
4372 		struct list_head *head = &lrugen->folios[gen][type][zone];
4623 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4641 				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
4782 		if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL)
4805 	return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ?
4816 	struct lru_gen_folio *lrugen;
4828 	hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) {
4837 		if (gen != READ_ONCE(lrugen->gen))
4840 		lruvec = container_of(lrugen, struct lruvec, lrugen);
4970 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
4972 	if (lrugen->enabled) {
4983 			if (!list_empty(&lrugen->folios[gen][type][zone]))
5028 		struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
5082 			lruvec->lrugen.enabled = enabled;
5245 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5256 				n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
5257 				n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
5260 				n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
5261 				n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
5263 					n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);
5299 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5327 		unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
5336 				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
5565 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
5568 	lrugen->max_seq = MIN_NR_GENS + 1;
5569 	lrugen->enabled = lru_gen_enabled();
5572 		lrugen->timestamps[i] = jiffies;
5575 		INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
5606 		VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
5607 					   sizeof(lruvec->lrugen.nr_pages)));
5609 		lruvec->lrugen.list.next = LIST_POISON1;