checkpoint.c revision 52118743
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/checkpoint.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 *             http://www.samsung.com/
7 */
8#include <linux/fs.h>
9#include <linux/bio.h>
10#include <linux/mpage.h>
11#include <linux/writeback.h>
12#include <linux/blkdev.h>
13#include <linux/f2fs_fs.h>
14#include <linux/pagevec.h>
15#include <linux/swap.h>
16#include <linux/kthread.h>
17
18#include "f2fs.h"
19#include "node.h"
20#include "segment.h"
21#include "iostat.h"
22#include <trace/events/f2fs.h>
23
24#define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3))
25
26static struct kmem_cache *ino_entry_slab;
27struct kmem_cache *f2fs_inode_entry_slab;
28
29void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io)
30{
31	f2fs_build_fault_attr(sbi, 0, 0);
32	set_ckpt_flags(sbi, CP_ERROR_FLAG);
33	if (!end_io)
34		f2fs_flush_merged_writes(sbi);
35}
36
37/*
38 * We guarantee no failure on the returned page.
39 */
40struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
41{
42	struct address_space *mapping = META_MAPPING(sbi);
43	struct page *page;
44repeat:
45	page = f2fs_grab_cache_page(mapping, index, false);
46	if (!page) {
47		cond_resched();
48		goto repeat;
49	}
50	f2fs_wait_on_page_writeback(page, META, true, true);
51	if (!PageUptodate(page))
52		SetPageUptodate(page);
53	return page;
54}
55
56static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index,
57							bool is_meta)
58{
59	struct address_space *mapping = META_MAPPING(sbi);
60	struct page *page;
61	struct f2fs_io_info fio = {
62		.sbi = sbi,
63		.type = META,
64		.op = REQ_OP_READ,
65		.op_flags = REQ_META | REQ_PRIO,
66		.old_blkaddr = index,
67		.new_blkaddr = index,
68		.encrypted_page = NULL,
69		.is_por = !is_meta,
70	};
71	int err;
72
73	if (unlikely(!is_meta))
74		fio.op_flags &= ~REQ_META;
75repeat:
76	page = f2fs_grab_cache_page(mapping, index, false);
77	if (!page) {
78		cond_resched();
79		goto repeat;
80	}
81	if (PageUptodate(page))
82		goto out;
83
84	fio.page = page;
85
86	err = f2fs_submit_page_bio(&fio);
87	if (err) {
88		f2fs_put_page(page, 1);
89		return ERR_PTR(err);
90	}
91
92	f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
93
94	lock_page(page);
95	if (unlikely(page->mapping != mapping)) {
96		f2fs_put_page(page, 1);
97		goto repeat;
98	}
99
100	if (unlikely(!PageUptodate(page))) {
101		f2fs_put_page(page, 1);
102		return ERR_PTR(-EIO);
103	}
104out:
105	return page;
106}
107
108struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
109{
110	return __get_meta_page(sbi, index, true);
111}
112
113struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index)
114{
115	struct page *page;
116	int count = 0;
117
118retry:
119	page = __get_meta_page(sbi, index, true);
120	if (IS_ERR(page)) {
121		if (PTR_ERR(page) == -EIO &&
122				++count <= DEFAULT_RETRY_IO_COUNT)
123			goto retry;
124		f2fs_stop_checkpoint(sbi, false);
125	}
126	return page;
127}
128
129/* for POR only */
130struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index)
131{
132	return __get_meta_page(sbi, index, false);
133}
134
135static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr,
136							int type)
137{
138	struct seg_entry *se;
139	unsigned int segno, offset;
140	bool exist;
141
142	if (type != DATA_GENERIC_ENHANCE && type != DATA_GENERIC_ENHANCE_READ)
143		return true;
144
145	segno = GET_SEGNO(sbi, blkaddr);
146	offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
147	se = get_seg_entry(sbi, segno);
148
149	exist = f2fs_test_bit(offset, se->cur_valid_map);
150	if (!exist && type == DATA_GENERIC_ENHANCE) {
151		f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
152			 blkaddr, exist);
153		set_sbi_flag(sbi, SBI_NEED_FSCK);
154		WARN_ON(1);
155	}
156	return exist;
157}
158
159bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
160					block_t blkaddr, int type)
161{
162	switch (type) {
163	case META_NAT:
164		break;
165	case META_SIT:
166		if (unlikely(blkaddr >= SIT_BLK_CNT(sbi)))
167			return false;
168		break;
169	case META_SSA:
170		if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) ||
171			blkaddr < SM_I(sbi)->ssa_blkaddr))
172			return false;
173		break;
174	case META_CP:
175		if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr ||
176			blkaddr < __start_cp_addr(sbi)))
177			return false;
178		break;
179	case META_POR:
180		if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
181			blkaddr < MAIN_BLKADDR(sbi)))
182			return false;
183		break;
184	case DATA_GENERIC:
185	case DATA_GENERIC_ENHANCE:
186	case DATA_GENERIC_ENHANCE_READ:
187		if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
188				blkaddr < MAIN_BLKADDR(sbi))) {
189			f2fs_warn(sbi, "access invalid blkaddr:%u",
190				  blkaddr);
191			set_sbi_flag(sbi, SBI_NEED_FSCK);
192			WARN_ON(1);
193			return false;
194		} else {
195			return __is_bitmap_valid(sbi, blkaddr, type);
196		}
197		break;
198	case META_GENERIC:
199		if (unlikely(blkaddr < SEG0_BLKADDR(sbi) ||
200			blkaddr >= MAIN_BLKADDR(sbi)))
201			return false;
202		break;
203	default:
204		BUG();
205	}
206
207	return true;
208}
209
210/*
211 * Readahead CP/NAT/SIT/SSA/POR pages
212 */
213int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
214							int type, bool sync)
215{
216	struct page *page;
217	block_t blkno = start;
218	struct f2fs_io_info fio = {
219		.sbi = sbi,
220		.type = META,
221		.op = REQ_OP_READ,
222		.op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
223		.encrypted_page = NULL,
224		.in_list = false,
225		.is_por = (type == META_POR),
226	};
227	struct blk_plug plug;
228	int err;
229
230	if (unlikely(type == META_POR))
231		fio.op_flags &= ~REQ_META;
232
233	blk_start_plug(&plug);
234	for (; nrpages-- > 0; blkno++) {
235
236		if (!f2fs_is_valid_blkaddr(sbi, blkno, type))
237			goto out;
238
239		switch (type) {
240		case META_NAT:
241			if (unlikely(blkno >=
242					NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
243				blkno = 0;
244			/* get nat block addr */
245			fio.new_blkaddr = current_nat_addr(sbi,
246					blkno * NAT_ENTRY_PER_BLOCK);
247			break;
248		case META_SIT:
249			if (unlikely(blkno >= TOTAL_SEGS(sbi)))
250				goto out;
251			/* get sit block addr */
252			fio.new_blkaddr = current_sit_addr(sbi,
253					blkno * SIT_ENTRY_PER_BLOCK);
254			break;
255		case META_SSA:
256		case META_CP:
257		case META_POR:
258			fio.new_blkaddr = blkno;
259			break;
260		default:
261			BUG();
262		}
263
264		page = f2fs_grab_cache_page(META_MAPPING(sbi),
265						fio.new_blkaddr, false);
266		if (!page)
267			continue;
268		if (PageUptodate(page)) {
269			f2fs_put_page(page, 1);
270			continue;
271		}
272
273		fio.page = page;
274		err = f2fs_submit_page_bio(&fio);
275		f2fs_put_page(page, err ? 1 : 0);
276
277		if (!err)
278			f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
279	}
280out:
281	blk_finish_plug(&plug);
282	return blkno - start;
283}
284
285void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index)
286{
287	struct page *page;
288	bool readahead = false;
289
290	page = find_get_page(META_MAPPING(sbi), index);
291	if (!page || !PageUptodate(page))
292		readahead = true;
293	f2fs_put_page(page, 0);
294
295	if (readahead)
296		f2fs_ra_meta_pages(sbi, index, BIO_MAX_VECS, META_POR, true);
297}
298
299static int __f2fs_write_meta_page(struct page *page,
300				struct writeback_control *wbc,
301				enum iostat_type io_type)
302{
303	struct f2fs_sb_info *sbi = F2FS_P_SB(page);
304
305	trace_f2fs_writepage(page, META);
306
307	if (unlikely(f2fs_cp_error(sbi)))
308		goto redirty_out;
309	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
310		goto redirty_out;
311	if (wbc->for_reclaim && page->index < GET_SUM_BLOCK(sbi, 0))
312		goto redirty_out;
313
314	f2fs_do_write_meta_page(sbi, page, io_type);
315	dec_page_count(sbi, F2FS_DIRTY_META);
316
317	if (wbc->for_reclaim)
318		f2fs_submit_merged_write_cond(sbi, NULL, page, 0, META);
319
320	unlock_page(page);
321
322	if (unlikely(f2fs_cp_error(sbi)))
323		f2fs_submit_merged_write(sbi, META);
324
325	return 0;
326
327redirty_out:
328	redirty_page_for_writepage(wbc, page);
329	return AOP_WRITEPAGE_ACTIVATE;
330}
331
332static int f2fs_write_meta_page(struct page *page,
333				struct writeback_control *wbc)
334{
335	return __f2fs_write_meta_page(page, wbc, FS_META_IO);
336}
337
338static int f2fs_write_meta_pages(struct address_space *mapping,
339				struct writeback_control *wbc)
340{
341	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
342	long diff, written;
343
344	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
345		goto skip_write;
346
347	/* collect a number of dirty meta pages and write together */
348	if (wbc->sync_mode != WB_SYNC_ALL &&
349			get_pages(sbi, F2FS_DIRTY_META) <
350					nr_pages_to_skip(sbi, META))
351		goto skip_write;
352
353	/* if locked failed, cp will flush dirty pages instead */
354	if (!down_write_trylock(&sbi->cp_global_sem))
355		goto skip_write;
356
357	trace_f2fs_writepages(mapping->host, wbc, META);
358	diff = nr_pages_to_write(sbi, META, wbc);
359	written = f2fs_sync_meta_pages(sbi, META, wbc->nr_to_write, FS_META_IO);
360	up_write(&sbi->cp_global_sem);
361	wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
362	return 0;
363
364skip_write:
365	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
366	trace_f2fs_writepages(mapping->host, wbc, META);
367	return 0;
368}
369
370long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
371				long nr_to_write, enum iostat_type io_type)
372{
373	struct address_space *mapping = META_MAPPING(sbi);
374	pgoff_t index = 0, prev = ULONG_MAX;
375	struct pagevec pvec;
376	long nwritten = 0;
377	int nr_pages;
378	struct writeback_control wbc = {
379		.for_reclaim = 0,
380	};
381	struct blk_plug plug;
382
383	pagevec_init(&pvec);
384
385	blk_start_plug(&plug);
386
387	while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
388				PAGECACHE_TAG_DIRTY))) {
389		int i;
390
391		for (i = 0; i < nr_pages; i++) {
392			struct page *page = pvec.pages[i];
393
394			if (prev == ULONG_MAX)
395				prev = page->index - 1;
396			if (nr_to_write != LONG_MAX && page->index != prev + 1) {
397				pagevec_release(&pvec);
398				goto stop;
399			}
400
401			lock_page(page);
402
403			if (unlikely(page->mapping != mapping)) {
404continue_unlock:
405				unlock_page(page);
406				continue;
407			}
408			if (!PageDirty(page)) {
409				/* someone wrote it for us */
410				goto continue_unlock;
411			}
412
413			f2fs_wait_on_page_writeback(page, META, true, true);
414
415			if (!clear_page_dirty_for_io(page))
416				goto continue_unlock;
417
418			if (__f2fs_write_meta_page(page, &wbc, io_type)) {
419				unlock_page(page);
420				break;
421			}
422			nwritten++;
423			prev = page->index;
424			if (unlikely(nwritten >= nr_to_write))
425				break;
426		}
427		pagevec_release(&pvec);
428		cond_resched();
429	}
430stop:
431	if (nwritten)
432		f2fs_submit_merged_write(sbi, type);
433
434	blk_finish_plug(&plug);
435
436	return nwritten;
437}
438
439static int f2fs_set_meta_page_dirty(struct page *page)
440{
441	trace_f2fs_set_page_dirty(page, META);
442
443	if (!PageUptodate(page))
444		SetPageUptodate(page);
445	if (!PageDirty(page)) {
446		__set_page_dirty_nobuffers(page);
447		inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META);
448		set_page_private_reference(page);
449		return 1;
450	}
451	return 0;
452}
453
454const struct address_space_operations f2fs_meta_aops = {
455	.writepage	= f2fs_write_meta_page,
456	.writepages	= f2fs_write_meta_pages,
457	.set_page_dirty	= f2fs_set_meta_page_dirty,
458	.invalidatepage = f2fs_invalidate_page,
459	.releasepage	= f2fs_release_page,
460#ifdef CONFIG_MIGRATION
461	.migratepage    = f2fs_migrate_page,
462#endif
463};
464
465static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
466						unsigned int devidx, int type)
467{
468	struct inode_management *im = &sbi->im[type];
469	struct ino_entry *e = NULL, *new = NULL;
470
471	if (type == FLUSH_INO) {
472		rcu_read_lock();
473		e = radix_tree_lookup(&im->ino_root, ino);
474		rcu_read_unlock();
475	}
476
477retry:
478	if (!e)
479		new = f2fs_kmem_cache_alloc(ino_entry_slab,
480						GFP_NOFS, true, NULL);
481
482	radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
483
484	spin_lock(&im->ino_lock);
485	e = radix_tree_lookup(&im->ino_root, ino);
486	if (!e) {
487		if (!new) {
488			spin_unlock(&im->ino_lock);
489			goto retry;
490		}
491		e = new;
492		if (unlikely(radix_tree_insert(&im->ino_root, ino, e)))
493			f2fs_bug_on(sbi, 1);
494
495		memset(e, 0, sizeof(struct ino_entry));
496		e->ino = ino;
497
498		list_add_tail(&e->list, &im->ino_list);
499		if (type != ORPHAN_INO)
500			im->ino_num++;
501	}
502
503	if (type == FLUSH_INO)
504		f2fs_set_bit(devidx, (char *)&e->dirty_device);
505
506	spin_unlock(&im->ino_lock);
507	radix_tree_preload_end();
508
509	if (new && e != new)
510		kmem_cache_free(ino_entry_slab, new);
511}
512
513static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
514{
515	struct inode_management *im = &sbi->im[type];
516	struct ino_entry *e;
517
518	spin_lock(&im->ino_lock);
519	e = radix_tree_lookup(&im->ino_root, ino);
520	if (e) {
521		list_del(&e->list);
522		radix_tree_delete(&im->ino_root, ino);
523		im->ino_num--;
524		spin_unlock(&im->ino_lock);
525		kmem_cache_free(ino_entry_slab, e);
526		return;
527	}
528	spin_unlock(&im->ino_lock);
529}
530
531void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
532{
533	/* add new dirty ino entry into list */
534	__add_ino_entry(sbi, ino, 0, type);
535}
536
537void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
538{
539	/* remove dirty ino entry from list */
540	__remove_ino_entry(sbi, ino, type);
541}
542
543/* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */
544bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
545{
546	struct inode_management *im = &sbi->im[mode];
547	struct ino_entry *e;
548
549	spin_lock(&im->ino_lock);
550	e = radix_tree_lookup(&im->ino_root, ino);
551	spin_unlock(&im->ino_lock);
552	return e ? true : false;
553}
554
555void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
556{
557	struct ino_entry *e, *tmp;
558	int i;
559
560	for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) {
561		struct inode_management *im = &sbi->im[i];
562
563		spin_lock(&im->ino_lock);
564		list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
565			list_del(&e->list);
566			radix_tree_delete(&im->ino_root, e->ino);
567			kmem_cache_free(ino_entry_slab, e);
568			im->ino_num--;
569		}
570		spin_unlock(&im->ino_lock);
571	}
572}
573
574void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
575					unsigned int devidx, int type)
576{
577	__add_ino_entry(sbi, ino, devidx, type);
578}
579
580bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
581					unsigned int devidx, int type)
582{
583	struct inode_management *im = &sbi->im[type];
584	struct ino_entry *e;
585	bool is_dirty = false;
586
587	spin_lock(&im->ino_lock);
588	e = radix_tree_lookup(&im->ino_root, ino);
589	if (e && f2fs_test_bit(devidx, (char *)&e->dirty_device))
590		is_dirty = true;
591	spin_unlock(&im->ino_lock);
592	return is_dirty;
593}
594
595int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi)
596{
597	struct inode_management *im = &sbi->im[ORPHAN_INO];
598	int err = 0;
599
600	spin_lock(&im->ino_lock);
601
602	if (time_to_inject(sbi, FAULT_ORPHAN)) {
603		spin_unlock(&im->ino_lock);
604		f2fs_show_injection_info(sbi, FAULT_ORPHAN);
605		return -ENOSPC;
606	}
607
608	if (unlikely(im->ino_num >= sbi->max_orphans))
609		err = -ENOSPC;
610	else
611		im->ino_num++;
612	spin_unlock(&im->ino_lock);
613
614	return err;
615}
616
617void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi)
618{
619	struct inode_management *im = &sbi->im[ORPHAN_INO];
620
621	spin_lock(&im->ino_lock);
622	f2fs_bug_on(sbi, im->ino_num == 0);
623	im->ino_num--;
624	spin_unlock(&im->ino_lock);
625}
626
627void f2fs_add_orphan_inode(struct inode *inode)
628{
629	/* add new orphan ino entry into list */
630	__add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO);
631	f2fs_update_inode_page(inode);
632}
633
634void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
635{
636	/* remove orphan entry from orphan list */
637	__remove_ino_entry(sbi, ino, ORPHAN_INO);
638}
639
640static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
641{
642	struct inode *inode;
643	struct node_info ni;
644	int err;
645
646	inode = f2fs_iget_retry(sbi->sb, ino);
647	if (IS_ERR(inode)) {
648		/*
649		 * there should be a bug that we can't find the entry
650		 * to orphan inode.
651		 */
652		f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT);
653		return PTR_ERR(inode);
654	}
655
656	err = dquot_initialize(inode);
657	if (err) {
658		iput(inode);
659		goto err_out;
660	}
661
662	clear_nlink(inode);
663
664	/* truncate all the data during iput */
665	iput(inode);
666
667	err = f2fs_get_node_info(sbi, ino, &ni);
668	if (err)
669		goto err_out;
670
671	/* ENOMEM was fully retried in f2fs_evict_inode. */
672	if (ni.blk_addr != NULL_ADDR) {
673		err = -EIO;
674		goto err_out;
675	}
676	return 0;
677
678err_out:
679	set_sbi_flag(sbi, SBI_NEED_FSCK);
680	f2fs_warn(sbi, "%s: orphan failed (ino=%x), run fsck to fix.",
681		  __func__, ino);
682	return err;
683}
684
685int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
686{
687	block_t start_blk, orphan_blocks, i, j;
688	unsigned int s_flags = sbi->sb->s_flags;
689	int err = 0;
690#ifdef CONFIG_QUOTA
691	int quota_enabled;
692#endif
693
694	if (!is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG))
695		return 0;
696
697	if (bdev_read_only(sbi->sb->s_bdev)) {
698		f2fs_info(sbi, "write access unavailable, skipping orphan cleanup");
699		return 0;
700	}
701
702	if (s_flags & SB_RDONLY) {
703		f2fs_info(sbi, "orphan cleanup on readonly fs");
704		sbi->sb->s_flags &= ~SB_RDONLY;
705	}
706
707#ifdef CONFIG_QUOTA
708	/* Needed for iput() to work correctly and not trash data */
709	sbi->sb->s_flags |= SB_ACTIVE;
710
711	/*
712	 * Turn on quotas which were not enabled for read-only mounts if
713	 * filesystem has quota feature, so that they are updated correctly.
714	 */
715	quota_enabled = f2fs_enable_quota_files(sbi, s_flags & SB_RDONLY);
716#endif
717
718	start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi);
719	orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi);
720
721	f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true);
722
723	for (i = 0; i < orphan_blocks; i++) {
724		struct page *page;
725		struct f2fs_orphan_block *orphan_blk;
726
727		page = f2fs_get_meta_page(sbi, start_blk + i);
728		if (IS_ERR(page)) {
729			err = PTR_ERR(page);
730			goto out;
731		}
732
733		orphan_blk = (struct f2fs_orphan_block *)page_address(page);
734		for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
735			nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
736
737			err = recover_orphan_inode(sbi, ino);
738			if (err) {
739				f2fs_put_page(page, 1);
740				goto out;
741			}
742		}
743		f2fs_put_page(page, 1);
744	}
745	/* clear Orphan Flag */
746	clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG);
747out:
748	set_sbi_flag(sbi, SBI_IS_RECOVERED);
749
750#ifdef CONFIG_QUOTA
751	/* Turn quotas off */
752	if (quota_enabled)
753		f2fs_quota_off_umount(sbi->sb);
754#endif
755	sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */
756
757	return err;
758}
759
760static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
761{
762	struct list_head *head;
763	struct f2fs_orphan_block *orphan_blk = NULL;
764	unsigned int nentries = 0;
765	unsigned short index = 1;
766	unsigned short orphan_blocks;
767	struct page *page = NULL;
768	struct ino_entry *orphan = NULL;
769	struct inode_management *im = &sbi->im[ORPHAN_INO];
770
771	orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
772
773	/*
774	 * we don't need to do spin_lock(&im->ino_lock) here, since all the
775	 * orphan inode operations are covered under f2fs_lock_op().
776	 * And, spin_lock should be avoided due to page operations below.
777	 */
778	head = &im->ino_list;
779
780	/* loop for each orphan inode entry and write them in Jornal block */
781	list_for_each_entry(orphan, head, list) {
782		if (!page) {
783			page = f2fs_grab_meta_page(sbi, start_blk++);
784			orphan_blk =
785				(struct f2fs_orphan_block *)page_address(page);
786			memset(orphan_blk, 0, sizeof(*orphan_blk));
787		}
788
789		orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
790
791		if (nentries == F2FS_ORPHANS_PER_BLOCK) {
792			/*
793			 * an orphan block is full of 1020 entries,
794			 * then we need to flush current orphan blocks
795			 * and bring another one in memory
796			 */
797			orphan_blk->blk_addr = cpu_to_le16(index);
798			orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
799			orphan_blk->entry_count = cpu_to_le32(nentries);
800			set_page_dirty(page);
801			f2fs_put_page(page, 1);
802			index++;
803			nentries = 0;
804			page = NULL;
805		}
806	}
807
808	if (page) {
809		orphan_blk->blk_addr = cpu_to_le16(index);
810		orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
811		orphan_blk->entry_count = cpu_to_le32(nentries);
812		set_page_dirty(page);
813		f2fs_put_page(page, 1);
814	}
815}
816
817static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi,
818						struct f2fs_checkpoint *ckpt)
819{
820	unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset);
821	__u32 chksum;
822
823	chksum = f2fs_crc32(sbi, ckpt, chksum_ofs);
824	if (chksum_ofs < CP_CHKSUM_OFFSET) {
825		chksum_ofs += sizeof(chksum);
826		chksum = f2fs_chksum(sbi, chksum, (__u8 *)ckpt + chksum_ofs,
827						F2FS_BLKSIZE - chksum_ofs);
828	}
829	return chksum;
830}
831
832static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
833		struct f2fs_checkpoint **cp_block, struct page **cp_page,
834		unsigned long long *version)
835{
836	size_t crc_offset = 0;
837	__u32 crc;
838
839	*cp_page = f2fs_get_meta_page(sbi, cp_addr);
840	if (IS_ERR(*cp_page))
841		return PTR_ERR(*cp_page);
842
843	*cp_block = (struct f2fs_checkpoint *)page_address(*cp_page);
844
845	crc_offset = le32_to_cpu((*cp_block)->checksum_offset);
846	if (crc_offset < CP_MIN_CHKSUM_OFFSET ||
847			crc_offset > CP_CHKSUM_OFFSET) {
848		f2fs_put_page(*cp_page, 1);
849		f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset);
850		return -EINVAL;
851	}
852
853	crc = f2fs_checkpoint_chksum(sbi, *cp_block);
854	if (crc != cur_cp_crc(*cp_block)) {
855		f2fs_put_page(*cp_page, 1);
856		f2fs_warn(sbi, "invalid crc value");
857		return -EINVAL;
858	}
859
860	*version = cur_cp_version(*cp_block);
861	return 0;
862}
863
864static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
865				block_t cp_addr, unsigned long long *version)
866{
867	struct page *cp_page_1 = NULL, *cp_page_2 = NULL;
868	struct f2fs_checkpoint *cp_block = NULL;
869	unsigned long long cur_version = 0, pre_version = 0;
870	int err;
871
872	err = get_checkpoint_version(sbi, cp_addr, &cp_block,
873					&cp_page_1, version);
874	if (err)
875		return NULL;
876
877	if (le32_to_cpu(cp_block->cp_pack_total_block_count) >
878					sbi->blocks_per_seg) {
879		f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u",
880			  le32_to_cpu(cp_block->cp_pack_total_block_count));
881		goto invalid_cp;
882	}
883	pre_version = *version;
884
885	cp_addr += le32_to_cpu(cp_block->cp_pack_total_block_count) - 1;
886	err = get_checkpoint_version(sbi, cp_addr, &cp_block,
887					&cp_page_2, version);
888	if (err)
889		goto invalid_cp;
890	cur_version = *version;
891
892	if (cur_version == pre_version) {
893		*version = cur_version;
894		f2fs_put_page(cp_page_2, 1);
895		return cp_page_1;
896	}
897	f2fs_put_page(cp_page_2, 1);
898invalid_cp:
899	f2fs_put_page(cp_page_1, 1);
900	return NULL;
901}
902
903int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
904{
905	struct f2fs_checkpoint *cp_block;
906	struct f2fs_super_block *fsb = sbi->raw_super;
907	struct page *cp1, *cp2, *cur_page;
908	unsigned long blk_size = sbi->blocksize;
909	unsigned long long cp1_version = 0, cp2_version = 0;
910	unsigned long long cp_start_blk_no;
911	unsigned int cp_blks = 1 + __cp_payload(sbi);
912	block_t cp_blk_no;
913	int i;
914	int err;
915
916	sbi->ckpt = f2fs_kvzalloc(sbi, array_size(blk_size, cp_blks),
917				  GFP_KERNEL);
918	if (!sbi->ckpt)
919		return -ENOMEM;
920	/*
921	 * Finding out valid cp block involves read both
922	 * sets( cp pack 1 and cp pack 2)
923	 */
924	cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
925	cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);
926
927	/* The second checkpoint pack should start at the next segment */
928	cp_start_blk_no += ((unsigned long long)1) <<
929				le32_to_cpu(fsb->log_blocks_per_seg);
930	cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);
931
932	if (cp1 && cp2) {
933		if (ver_after(cp2_version, cp1_version))
934			cur_page = cp2;
935		else
936			cur_page = cp1;
937	} else if (cp1) {
938		cur_page = cp1;
939	} else if (cp2) {
940		cur_page = cp2;
941	} else {
942		err = -EFSCORRUPTED;
943		goto fail_no_cp;
944	}
945
946	cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
947	memcpy(sbi->ckpt, cp_block, blk_size);
948
949	if (cur_page == cp1)
950		sbi->cur_cp_pack = 1;
951	else
952		sbi->cur_cp_pack = 2;
953
954	/* Sanity checking of checkpoint */
955	if (f2fs_sanity_check_ckpt(sbi)) {
956		err = -EFSCORRUPTED;
957		goto free_fail_no_cp;
958	}
959
960	if (cp_blks <= 1)
961		goto done;
962
963	cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
964	if (cur_page == cp2)
965		cp_blk_no += 1 << le32_to_cpu(fsb->log_blocks_per_seg);
966
967	for (i = 1; i < cp_blks; i++) {
968		void *sit_bitmap_ptr;
969		unsigned char *ckpt = (unsigned char *)sbi->ckpt;
970
971		cur_page = f2fs_get_meta_page(sbi, cp_blk_no + i);
972		if (IS_ERR(cur_page)) {
973			err = PTR_ERR(cur_page);
974			goto free_fail_no_cp;
975		}
976		sit_bitmap_ptr = page_address(cur_page);
977		memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
978		f2fs_put_page(cur_page, 1);
979	}
980done:
981	f2fs_put_page(cp1, 1);
982	f2fs_put_page(cp2, 1);
983	return 0;
984
985free_fail_no_cp:
986	f2fs_put_page(cp1, 1);
987	f2fs_put_page(cp2, 1);
988fail_no_cp:
989	kvfree(sbi->ckpt);
990	return err;
991}
992
993static void __add_dirty_inode(struct inode *inode, enum inode_type type)
994{
995	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
996	int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
997
998	if (is_inode_flag_set(inode, flag))
999		return;
1000
1001	set_inode_flag(inode, flag);
1002	if (!f2fs_is_volatile_file(inode))
1003		list_add_tail(&F2FS_I(inode)->dirty_list,
1004						&sbi->inode_list[type]);
1005	stat_inc_dirty_inode(sbi, type);
1006}
1007
1008static void __remove_dirty_inode(struct inode *inode, enum inode_type type)
1009{
1010	int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1011
1012	if (get_dirty_pages(inode) || !is_inode_flag_set(inode, flag))
1013		return;
1014
1015	list_del_init(&F2FS_I(inode)->dirty_list);
1016	clear_inode_flag(inode, flag);
1017	stat_dec_dirty_inode(F2FS_I_SB(inode), type);
1018}
1019
1020void f2fs_update_dirty_page(struct inode *inode, struct page *page)
1021{
1022	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1023	enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1024
1025	if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1026			!S_ISLNK(inode->i_mode))
1027		return;
1028
1029	spin_lock(&sbi->inode_lock[type]);
1030	if (type != FILE_INODE || test_opt(sbi, DATA_FLUSH))
1031		__add_dirty_inode(inode, type);
1032	inode_inc_dirty_pages(inode);
1033	spin_unlock(&sbi->inode_lock[type]);
1034
1035	set_page_private_reference(page);
1036}
1037
1038void f2fs_remove_dirty_inode(struct inode *inode)
1039{
1040	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1041	enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1042
1043	if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1044			!S_ISLNK(inode->i_mode))
1045		return;
1046
1047	if (type == FILE_INODE && !test_opt(sbi, DATA_FLUSH))
1048		return;
1049
1050	spin_lock(&sbi->inode_lock[type]);
1051	__remove_dirty_inode(inode, type);
1052	spin_unlock(&sbi->inode_lock[type]);
1053}
1054
1055int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type)
1056{
1057	struct list_head *head;
1058	struct inode *inode;
1059	struct f2fs_inode_info *fi;
1060	bool is_dir = (type == DIR_INODE);
1061	unsigned long ino = 0;
1062
1063	trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir,
1064				get_pages(sbi, is_dir ?
1065				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1066retry:
1067	if (unlikely(f2fs_cp_error(sbi))) {
1068		trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1069				get_pages(sbi, is_dir ?
1070				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1071		return -EIO;
1072	}
1073
1074	spin_lock(&sbi->inode_lock[type]);
1075
1076	head = &sbi->inode_list[type];
1077	if (list_empty(head)) {
1078		spin_unlock(&sbi->inode_lock[type]);
1079		trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1080				get_pages(sbi, is_dir ?
1081				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1082		return 0;
1083	}
1084	fi = list_first_entry(head, struct f2fs_inode_info, dirty_list);
1085	inode = igrab(&fi->vfs_inode);
1086	spin_unlock(&sbi->inode_lock[type]);
1087	if (inode) {
1088		unsigned long cur_ino = inode->i_ino;
1089
1090		F2FS_I(inode)->cp_task = current;
1091
1092		filemap_fdatawrite(inode->i_mapping);
1093
1094		F2FS_I(inode)->cp_task = NULL;
1095
1096		iput(inode);
1097		/* We need to give cpu to another writers. */
1098		if (ino == cur_ino)
1099			cond_resched();
1100		else
1101			ino = cur_ino;
1102	} else {
1103		/*
1104		 * We should submit bio, since it exists several
1105		 * wribacking dentry pages in the freeing inode.
1106		 */
1107		f2fs_submit_merged_write(sbi, DATA);
1108		cond_resched();
1109	}
1110	goto retry;
1111}
1112
1113int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
1114{
1115	struct list_head *head = &sbi->inode_list[DIRTY_META];
1116	struct inode *inode;
1117	struct f2fs_inode_info *fi;
1118	s64 total = get_pages(sbi, F2FS_DIRTY_IMETA);
1119
1120	while (total--) {
1121		if (unlikely(f2fs_cp_error(sbi)))
1122			return -EIO;
1123
1124		spin_lock(&sbi->inode_lock[DIRTY_META]);
1125		if (list_empty(head)) {
1126			spin_unlock(&sbi->inode_lock[DIRTY_META]);
1127			return 0;
1128		}
1129		fi = list_first_entry(head, struct f2fs_inode_info,
1130							gdirty_list);
1131		inode = igrab(&fi->vfs_inode);
1132		spin_unlock(&sbi->inode_lock[DIRTY_META]);
1133		if (inode) {
1134			sync_inode_metadata(inode, 0);
1135
1136			/* it's on eviction */
1137			if (is_inode_flag_set(inode, FI_DIRTY_INODE))
1138				f2fs_update_inode_page(inode);
1139			iput(inode);
1140		}
1141	}
1142	return 0;
1143}
1144
1145static void __prepare_cp_block(struct f2fs_sb_info *sbi)
1146{
1147	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1148	struct f2fs_nm_info *nm_i = NM_I(sbi);
1149	nid_t last_nid = nm_i->next_scan_nid;
1150
1151	next_free_nid(sbi, &last_nid);
1152	ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
1153	ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
1154	ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
1155	ckpt->next_free_nid = cpu_to_le32(last_nid);
1156}
1157
1158static bool __need_flush_quota(struct f2fs_sb_info *sbi)
1159{
1160	bool ret = false;
1161
1162	if (!is_journalled_quota(sbi))
1163		return false;
1164
1165	down_write(&sbi->quota_sem);
1166	if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
1167		ret = false;
1168	} else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
1169		ret = false;
1170	} else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
1171		clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1172		ret = true;
1173	} else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
1174		ret = true;
1175	}
1176	up_write(&sbi->quota_sem);
1177	return ret;
1178}
1179
1180/*
1181 * Freeze all the FS-operations for checkpoint.
1182 */
1183static int block_operations(struct f2fs_sb_info *sbi)
1184{
1185	struct writeback_control wbc = {
1186		.sync_mode = WB_SYNC_ALL,
1187		.nr_to_write = LONG_MAX,
1188		.for_reclaim = 0,
1189	};
1190	int err = 0, cnt = 0;
1191
1192	/*
1193	 * Let's flush inline_data in dirty node pages.
1194	 */
1195	f2fs_flush_inline_data(sbi);
1196
1197retry_flush_quotas:
1198	f2fs_lock_all(sbi);
1199	if (__need_flush_quota(sbi)) {
1200		int locked;
1201
1202		if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
1203			set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1204			set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1205			goto retry_flush_dents;
1206		}
1207		f2fs_unlock_all(sbi);
1208
1209		/* only failed during mount/umount/freeze/quotactl */
1210		locked = down_read_trylock(&sbi->sb->s_umount);
1211		f2fs_quota_sync(sbi->sb, -1);
1212		if (locked)
1213			up_read(&sbi->sb->s_umount);
1214		cond_resched();
1215		goto retry_flush_quotas;
1216	}
1217
1218retry_flush_dents:
1219	/* write all the dirty dentry pages */
1220	if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
1221		f2fs_unlock_all(sbi);
1222		err = f2fs_sync_dirty_inodes(sbi, DIR_INODE);
1223		if (err)
1224			return err;
1225		cond_resched();
1226		goto retry_flush_quotas;
1227	}
1228
1229	/*
1230	 * POR: we should ensure that there are no dirty node pages
1231	 * until finishing nat/sit flush. inode->i_blocks can be updated.
1232	 */
1233	down_write(&sbi->node_change);
1234
1235	if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
1236		up_write(&sbi->node_change);
1237		f2fs_unlock_all(sbi);
1238		err = f2fs_sync_inode_meta(sbi);
1239		if (err)
1240			return err;
1241		cond_resched();
1242		goto retry_flush_quotas;
1243	}
1244
1245retry_flush_nodes:
1246	down_write(&sbi->node_write);
1247
1248	if (get_pages(sbi, F2FS_DIRTY_NODES)) {
1249		up_write(&sbi->node_write);
1250		atomic_inc(&sbi->wb_sync_req[NODE]);
1251		err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO);
1252		atomic_dec(&sbi->wb_sync_req[NODE]);
1253		if (err) {
1254			up_write(&sbi->node_change);
1255			f2fs_unlock_all(sbi);
1256			return err;
1257		}
1258		cond_resched();
1259		goto retry_flush_nodes;
1260	}
1261
1262	/*
1263	 * sbi->node_change is used only for AIO write_begin path which produces
1264	 * dirty node blocks and some checkpoint values by block allocation.
1265	 */
1266	__prepare_cp_block(sbi);
1267	up_write(&sbi->node_change);
1268	return err;
1269}
1270
1271static void unblock_operations(struct f2fs_sb_info *sbi)
1272{
1273	up_write(&sbi->node_write);
1274	f2fs_unlock_all(sbi);
1275}
1276
1277void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
1278{
1279	DEFINE_WAIT(wait);
1280
1281	for (;;) {
1282		if (!get_pages(sbi, type))
1283			break;
1284
1285		if (unlikely(f2fs_cp_error(sbi)))
1286			break;
1287
1288		if (type == F2FS_DIRTY_META)
1289			f2fs_sync_meta_pages(sbi, META, LONG_MAX,
1290							FS_CP_META_IO);
1291		else if (type == F2FS_WB_CP_DATA)
1292			f2fs_submit_merged_write(sbi, DATA);
1293
1294		prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
1295		io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1296	}
1297	finish_wait(&sbi->cp_wait, &wait);
1298}
1299
1300static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1301{
1302	unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num;
1303	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1304	unsigned long flags;
1305
1306	spin_lock_irqsave(&sbi->cp_lock, flags);
1307
1308	if ((cpc->reason & CP_UMOUNT) &&
1309			le32_to_cpu(ckpt->cp_pack_total_block_count) >
1310			sbi->blocks_per_seg - NM_I(sbi)->nat_bits_blocks)
1311		disable_nat_bits(sbi, false);
1312
1313	if (cpc->reason & CP_TRIMMED)
1314		__set_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1315	else
1316		__clear_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1317
1318	if (cpc->reason & CP_UMOUNT)
1319		__set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1320	else
1321		__clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1322
1323	if (cpc->reason & CP_FASTBOOT)
1324		__set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1325	else
1326		__clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1327
1328	if (orphan_num)
1329		__set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1330	else
1331		__clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1332
1333	if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1334		__set_ckpt_flags(ckpt, CP_FSCK_FLAG);
1335
1336	if (is_sbi_flag_set(sbi, SBI_IS_RESIZEFS))
1337		__set_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1338	else
1339		__clear_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1340
1341	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
1342		__set_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1343	else
1344		__clear_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1345
1346	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK))
1347		__set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1348	else
1349		__clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1350
1351	if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
1352		__set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1353	else
1354		__clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1355
1356	if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
1357		__set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1358
1359	/* set this flag to activate crc|cp_ver for recovery */
1360	__set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG);
1361	__clear_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG);
1362
1363	spin_unlock_irqrestore(&sbi->cp_lock, flags);
1364}
1365
1366static void commit_checkpoint(struct f2fs_sb_info *sbi,
1367	void *src, block_t blk_addr)
1368{
1369	struct writeback_control wbc = {
1370		.for_reclaim = 0,
1371	};
1372
1373	/*
1374	 * pagevec_lookup_tag and lock_page again will take
1375	 * some extra time. Therefore, f2fs_update_meta_pages and
1376	 * f2fs_sync_meta_pages are combined in this function.
1377	 */
1378	struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
1379	int err;
1380
1381	f2fs_wait_on_page_writeback(page, META, true, true);
1382
1383	memcpy(page_address(page), src, PAGE_SIZE);
1384
1385	set_page_dirty(page);
1386	if (unlikely(!clear_page_dirty_for_io(page)))
1387		f2fs_bug_on(sbi, 1);
1388
1389	/* writeout cp pack 2 page */
1390	err = __f2fs_write_meta_page(page, &wbc, FS_CP_META_IO);
1391	if (unlikely(err && f2fs_cp_error(sbi))) {
1392		f2fs_put_page(page, 1);
1393		return;
1394	}
1395
1396	f2fs_bug_on(sbi, err);
1397	f2fs_put_page(page, 0);
1398
1399	/* submit checkpoint (with barrier if NOBARRIER is not set) */
1400	f2fs_submit_merged_write(sbi, META_FLUSH);
1401}
1402
1403static inline u64 get_sectors_written(struct block_device *bdev)
1404{
1405	return (u64)part_stat_read(bdev, sectors[STAT_WRITE]);
1406}
1407
1408u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi)
1409{
1410	if (f2fs_is_multi_device(sbi)) {
1411		u64 sectors = 0;
1412		int i;
1413
1414		for (i = 0; i < sbi->s_ndevs; i++)
1415			sectors += get_sectors_written(FDEV(i).bdev);
1416
1417		return sectors;
1418	}
1419
1420	return get_sectors_written(sbi->sb->s_bdev);
1421}
1422
1423static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1424{
1425	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1426	struct f2fs_nm_info *nm_i = NM_I(sbi);
1427	unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num, flags;
1428	block_t start_blk;
1429	unsigned int data_sum_blocks, orphan_blocks;
1430	__u32 crc32 = 0;
1431	int i;
1432	int cp_payload_blks = __cp_payload(sbi);
1433	struct curseg_info *seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE);
1434	u64 kbytes_written;
1435	int err;
1436
1437	/* Flush all the NAT/SIT pages */
1438	f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1439
1440	/* start to update checkpoint, cp ver is already updated previously */
1441	ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true));
1442	ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
1443	for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) {
1444		ckpt->cur_node_segno[i] =
1445			cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE));
1446		ckpt->cur_node_blkoff[i] =
1447			cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE));
1448		ckpt->alloc_type[i + CURSEG_HOT_NODE] =
1449				curseg_alloc_type(sbi, i + CURSEG_HOT_NODE);
1450	}
1451	for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) {
1452		ckpt->cur_data_segno[i] =
1453			cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA));
1454		ckpt->cur_data_blkoff[i] =
1455			cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA));
1456		ckpt->alloc_type[i + CURSEG_HOT_DATA] =
1457				curseg_alloc_type(sbi, i + CURSEG_HOT_DATA);
1458	}
1459
1460	/* 2 cp + n data seg summary + orphan inode blocks */
1461	data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false);
1462	spin_lock_irqsave(&sbi->cp_lock, flags);
1463	if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
1464		__set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1465	else
1466		__clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1467	spin_unlock_irqrestore(&sbi->cp_lock, flags);
1468
1469	orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
1470	ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
1471			orphan_blocks);
1472
1473	if (__remain_node_summaries(cpc->reason))
1474		ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1475				cp_payload_blks + data_sum_blocks +
1476				orphan_blocks + NR_CURSEG_NODE_TYPE);
1477	else
1478		ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1479				cp_payload_blks + data_sum_blocks +
1480				orphan_blocks);
1481
1482	/* update ckpt flag for checkpoint */
1483	update_ckpt_flags(sbi, cpc);
1484
1485	/* update SIT/NAT bitmap */
1486	get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
1487	get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
1488
1489	crc32 = f2fs_checkpoint_chksum(sbi, ckpt);
1490	*((__le32 *)((unsigned char *)ckpt +
1491				le32_to_cpu(ckpt->checksum_offset)))
1492				= cpu_to_le32(crc32);
1493
1494	start_blk = __start_cp_next_addr(sbi);
1495
1496	/* write nat bits */
1497	if (enabled_nat_bits(sbi, cpc)) {
1498		__u64 cp_ver = cur_cp_version(ckpt);
1499		block_t blk;
1500
1501		cp_ver |= ((__u64)crc32 << 32);
1502		*(__le64 *)nm_i->nat_bits = cpu_to_le64(cp_ver);
1503
1504		blk = start_blk + sbi->blocks_per_seg - nm_i->nat_bits_blocks;
1505		for (i = 0; i < nm_i->nat_bits_blocks; i++)
1506			f2fs_update_meta_page(sbi, nm_i->nat_bits +
1507					(i << F2FS_BLKSIZE_BITS), blk + i);
1508	}
1509
1510	/* write out checkpoint buffer at block 0 */
1511	f2fs_update_meta_page(sbi, ckpt, start_blk++);
1512
1513	for (i = 1; i < 1 + cp_payload_blks; i++)
1514		f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE,
1515							start_blk++);
1516
1517	if (orphan_num) {
1518		write_orphan_inodes(sbi, start_blk);
1519		start_blk += orphan_blocks;
1520	}
1521
1522	f2fs_write_data_summaries(sbi, start_blk);
1523	start_blk += data_sum_blocks;
1524
1525	/* Record write statistics in the hot node summary */
1526	kbytes_written = sbi->kbytes_written;
1527	kbytes_written += (f2fs_get_sectors_written(sbi) -
1528				sbi->sectors_written_start) >> 1;
1529	seg_i->journal->info.kbytes_written = cpu_to_le64(kbytes_written);
1530
1531	if (__remain_node_summaries(cpc->reason)) {
1532		f2fs_write_node_summaries(sbi, start_blk);
1533		start_blk += NR_CURSEG_NODE_TYPE;
1534	}
1535
1536	/* update user_block_counts */
1537	sbi->last_valid_block_count = sbi->total_valid_block_count;
1538	percpu_counter_set(&sbi->alloc_valid_block_count, 0);
1539
1540	/* Here, we have one bio having CP pack except cp pack 2 page */
1541	f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1542	/* Wait for all dirty meta pages to be submitted for IO */
1543	f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META);
1544
1545	/* wait for previous submitted meta pages writeback */
1546	f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1547
1548	/* flush all device cache */
1549	err = f2fs_flush_device_cache(sbi);
1550	if (err)
1551		return err;
1552
1553	/* barrier and flush checkpoint cp pack 2 page if it can */
1554	commit_checkpoint(sbi, ckpt, start_blk);
1555	f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1556
1557	/*
1558	 * invalidate intermediate page cache borrowed from meta inode which are
1559	 * used for migration of encrypted, verity or compressed inode's blocks.
1560	 */
1561	if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) ||
1562		f2fs_sb_has_compression(sbi))
1563		invalidate_mapping_pages(META_MAPPING(sbi),
1564				MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1);
1565
1566	f2fs_release_ino_entry(sbi, false);
1567
1568	f2fs_reset_fsync_node_info(sbi);
1569
1570	clear_sbi_flag(sbi, SBI_IS_DIRTY);
1571	clear_sbi_flag(sbi, SBI_NEED_CP);
1572	clear_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1573
1574	spin_lock(&sbi->stat_lock);
1575	sbi->unusable_block_count = 0;
1576	spin_unlock(&sbi->stat_lock);
1577
1578	__set_cp_next_pack(sbi);
1579
1580	/*
1581	 * redirty superblock if metadata like node page or inode cache is
1582	 * updated during writing checkpoint.
1583	 */
1584	if (get_pages(sbi, F2FS_DIRTY_NODES) ||
1585			get_pages(sbi, F2FS_DIRTY_IMETA))
1586		set_sbi_flag(sbi, SBI_IS_DIRTY);
1587
1588	f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS));
1589
1590	return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0;
1591}
1592
1593int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1594{
1595	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1596	unsigned long long ckpt_ver;
1597	int err = 0;
1598
1599	if (f2fs_readonly(sbi->sb) || f2fs_hw_is_readonly(sbi))
1600		return -EROFS;
1601
1602	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
1603		if (cpc->reason != CP_PAUSE)
1604			return 0;
1605		f2fs_warn(sbi, "Start checkpoint disabled!");
1606	}
1607	if (cpc->reason != CP_RESIZE)
1608		down_write(&sbi->cp_global_sem);
1609
1610	if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) &&
1611		((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) ||
1612		((cpc->reason & CP_DISCARD) && !sbi->discard_blks)))
1613		goto out;
1614	if (unlikely(f2fs_cp_error(sbi))) {
1615		err = -EIO;
1616		goto out;
1617	}
1618
1619	trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops");
1620
1621	err = block_operations(sbi);
1622	if (err)
1623		goto out;
1624
1625	trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops");
1626
1627	f2fs_flush_merged_writes(sbi);
1628
1629	/* this is the case of multiple fstrims without any changes */
1630	if (cpc->reason & CP_DISCARD) {
1631		if (!f2fs_exist_trim_candidates(sbi, cpc)) {
1632			unblock_operations(sbi);
1633			goto out;
1634		}
1635
1636		if (NM_I(sbi)->nat_cnt[DIRTY_NAT] == 0 &&
1637				SIT_I(sbi)->dirty_sentries == 0 &&
1638				prefree_segments(sbi) == 0) {
1639			f2fs_flush_sit_entries(sbi, cpc);
1640			f2fs_clear_prefree_segments(sbi, cpc);
1641			unblock_operations(sbi);
1642			goto out;
1643		}
1644	}
1645
1646	/*
1647	 * update checkpoint pack index
1648	 * Increase the version number so that
1649	 * SIT entries and seg summaries are written at correct place
1650	 */
1651	ckpt_ver = cur_cp_version(ckpt);
1652	ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);
1653
1654	/* write cached NAT/SIT entries to NAT/SIT area */
1655	err = f2fs_flush_nat_entries(sbi, cpc);
1656	if (err) {
1657		f2fs_err(sbi, "f2fs_flush_nat_entries failed err:%d, stop checkpoint", err);
1658		f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1659		goto stop;
1660	}
1661
1662	f2fs_flush_sit_entries(sbi, cpc);
1663
1664	/* save inmem log status */
1665	f2fs_save_inmem_curseg(sbi);
1666
1667	err = do_checkpoint(sbi, cpc);
1668	if (err) {
1669		f2fs_err(sbi, "do_checkpoint failed err:%d, stop checkpoint", err);
1670		f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1671		f2fs_release_discard_addrs(sbi);
1672	} else {
1673		f2fs_clear_prefree_segments(sbi, cpc);
1674	}
1675
1676	f2fs_restore_inmem_curseg(sbi);
1677stop:
1678	unblock_operations(sbi);
1679	stat_inc_cp_count(sbi->stat_info);
1680
1681	if (cpc->reason & CP_RECOVERY)
1682		f2fs_notice(sbi, "checkpoint: version = %llx", ckpt_ver);
1683
1684	/* update CP_TIME to trigger checkpoint periodically */
1685	f2fs_update_time(sbi, CP_TIME);
1686	trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint");
1687out:
1688	if (cpc->reason != CP_RESIZE)
1689		up_write(&sbi->cp_global_sem);
1690	return err;
1691}
1692
1693void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi)
1694{
1695	int i;
1696
1697	for (i = 0; i < MAX_INO_ENTRY; i++) {
1698		struct inode_management *im = &sbi->im[i];
1699
1700		INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC);
1701		spin_lock_init(&im->ino_lock);
1702		INIT_LIST_HEAD(&im->ino_list);
1703		im->ino_num = 0;
1704	}
1705
1706	sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS -
1707			NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) *
1708				F2FS_ORPHANS_PER_BLOCK;
1709}
1710
1711int __init f2fs_create_checkpoint_caches(void)
1712{
1713	ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry",
1714			sizeof(struct ino_entry));
1715	if (!ino_entry_slab)
1716		return -ENOMEM;
1717	f2fs_inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry",
1718			sizeof(struct inode_entry));
1719	if (!f2fs_inode_entry_slab) {
1720		kmem_cache_destroy(ino_entry_slab);
1721		return -ENOMEM;
1722	}
1723	return 0;
1724}
1725
1726void f2fs_destroy_checkpoint_caches(void)
1727{
1728	kmem_cache_destroy(ino_entry_slab);
1729	kmem_cache_destroy(f2fs_inode_entry_slab);
1730}
1731
1732static int __write_checkpoint_sync(struct f2fs_sb_info *sbi)
1733{
1734	struct cp_control cpc = { .reason = CP_SYNC, };
1735	int err;
1736
1737	down_write(&sbi->gc_lock);
1738	err = f2fs_write_checkpoint(sbi, &cpc);
1739	up_write(&sbi->gc_lock);
1740
1741	return err;
1742}
1743
1744static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi)
1745{
1746	struct ckpt_req_control *cprc = &sbi->cprc_info;
1747	struct ckpt_req *req, *next;
1748	struct llist_node *dispatch_list;
1749	u64 sum_diff = 0, diff, count = 0;
1750	int ret;
1751
1752	dispatch_list = llist_del_all(&cprc->issue_list);
1753	if (!dispatch_list)
1754		return;
1755	dispatch_list = llist_reverse_order(dispatch_list);
1756
1757	ret = __write_checkpoint_sync(sbi);
1758	atomic_inc(&cprc->issued_ckpt);
1759
1760	llist_for_each_entry_safe(req, next, dispatch_list, llnode) {
1761		diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time);
1762		req->ret = ret;
1763		complete(&req->wait);
1764
1765		sum_diff += diff;
1766		count++;
1767	}
1768	atomic_sub(count, &cprc->queued_ckpt);
1769	atomic_add(count, &cprc->total_ckpt);
1770
1771	spin_lock(&cprc->stat_lock);
1772	cprc->cur_time = (unsigned int)div64_u64(sum_diff, count);
1773	if (cprc->peak_time < cprc->cur_time)
1774		cprc->peak_time = cprc->cur_time;
1775	spin_unlock(&cprc->stat_lock);
1776}
1777
1778static int issue_checkpoint_thread(void *data)
1779{
1780	struct f2fs_sb_info *sbi = data;
1781	struct ckpt_req_control *cprc = &sbi->cprc_info;
1782	wait_queue_head_t *q = &cprc->ckpt_wait_queue;
1783repeat:
1784	if (kthread_should_stop())
1785		return 0;
1786
1787	if (!llist_empty(&cprc->issue_list))
1788		__checkpoint_and_complete_reqs(sbi);
1789
1790	wait_event_interruptible(*q,
1791		kthread_should_stop() || !llist_empty(&cprc->issue_list));
1792	goto repeat;
1793}
1794
1795static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi,
1796		struct ckpt_req *wait_req)
1797{
1798	struct ckpt_req_control *cprc = &sbi->cprc_info;
1799
1800	if (!llist_empty(&cprc->issue_list)) {
1801		__checkpoint_and_complete_reqs(sbi);
1802	} else {
1803		/* already dispatched by issue_checkpoint_thread */
1804		if (wait_req)
1805			wait_for_completion(&wait_req->wait);
1806	}
1807}
1808
1809static void init_ckpt_req(struct ckpt_req *req)
1810{
1811	memset(req, 0, sizeof(struct ckpt_req));
1812
1813	init_completion(&req->wait);
1814	req->queue_time = ktime_get();
1815}
1816
1817int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi)
1818{
1819	struct ckpt_req_control *cprc = &sbi->cprc_info;
1820	struct ckpt_req req;
1821	struct cp_control cpc;
1822
1823	cpc.reason = __get_cp_reason(sbi);
1824	if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC) {
1825		int ret;
1826
1827		down_write(&sbi->gc_lock);
1828		ret = f2fs_write_checkpoint(sbi, &cpc);
1829		up_write(&sbi->gc_lock);
1830
1831		return ret;
1832	}
1833
1834	if (!cprc->f2fs_issue_ckpt)
1835		return __write_checkpoint_sync(sbi);
1836
1837	init_ckpt_req(&req);
1838
1839	llist_add(&req.llnode, &cprc->issue_list);
1840	atomic_inc(&cprc->queued_ckpt);
1841
1842	/*
1843	 * update issue_list before we wake up issue_checkpoint thread,
1844	 * this smp_mb() pairs with another barrier in ___wait_event(),
1845	 * see more details in comments of waitqueue_active().
1846	 */
1847	smp_mb();
1848
1849	if (waitqueue_active(&cprc->ckpt_wait_queue))
1850		wake_up(&cprc->ckpt_wait_queue);
1851
1852	if (cprc->f2fs_issue_ckpt)
1853		wait_for_completion(&req.wait);
1854	else
1855		flush_remained_ckpt_reqs(sbi, &req);
1856
1857	return req.ret;
1858}
1859
1860int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi)
1861{
1862	dev_t dev = sbi->sb->s_bdev->bd_dev;
1863	struct ckpt_req_control *cprc = &sbi->cprc_info;
1864
1865	if (cprc->f2fs_issue_ckpt)
1866		return 0;
1867
1868	cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi,
1869			"f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev));
1870	if (IS_ERR(cprc->f2fs_issue_ckpt)) {
1871		cprc->f2fs_issue_ckpt = NULL;
1872		return -ENOMEM;
1873	}
1874
1875	set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio);
1876
1877	return 0;
1878}
1879
1880void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi)
1881{
1882	struct ckpt_req_control *cprc = &sbi->cprc_info;
1883
1884	if (cprc->f2fs_issue_ckpt) {
1885		struct task_struct *ckpt_task = cprc->f2fs_issue_ckpt;
1886
1887		cprc->f2fs_issue_ckpt = NULL;
1888		kthread_stop(ckpt_task);
1889
1890		flush_remained_ckpt_reqs(sbi, NULL);
1891	}
1892}
1893
1894void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi)
1895{
1896	struct ckpt_req_control *cprc = &sbi->cprc_info;
1897
1898	atomic_set(&cprc->issued_ckpt, 0);
1899	atomic_set(&cprc->total_ckpt, 0);
1900	atomic_set(&cprc->queued_ckpt, 0);
1901	cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO;
1902	init_waitqueue_head(&cprc->ckpt_wait_queue);
1903	init_llist_head(&cprc->issue_list);
1904	spin_lock_init(&cprc->stat_lock);
1905}
1906