inode.c revision 188c299e
1// SPDX-License-Identifier: GPL-2.0
2/*
3 *  linux/fs/ext4/inode.c
4 *
5 * Copyright (C) 1992, 1993, 1994, 1995
6 * Remy Card (card@masi.ibp.fr)
7 * Laboratoire MASI - Institut Blaise Pascal
8 * Universite Pierre et Marie Curie (Paris VI)
9 *
10 *  from
11 *
12 *  linux/fs/minix/inode.c
13 *
14 *  Copyright (C) 1991, 1992  Linus Torvalds
15 *
16 *  64-bit file support on 64-bit platforms by Jakub Jelinek
17 *	(jj@sunsite.ms.mff.cuni.cz)
18 *
19 *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20 */
21
22#include <linux/fs.h>
23#include <linux/mount.h>
24#include <linux/time.h>
25#include <linux/highuid.h>
26#include <linux/pagemap.h>
27#include <linux/dax.h>
28#include <linux/quotaops.h>
29#include <linux/string.h>
30#include <linux/buffer_head.h>
31#include <linux/writeback.h>
32#include <linux/pagevec.h>
33#include <linux/mpage.h>
34#include <linux/namei.h>
35#include <linux/uio.h>
36#include <linux/bio.h>
37#include <linux/workqueue.h>
38#include <linux/kernel.h>
39#include <linux/printk.h>
40#include <linux/slab.h>
41#include <linux/bitops.h>
42#include <linux/iomap.h>
43#include <linux/iversion.h>
44
45#include "ext4_jbd2.h"
46#include "xattr.h"
47#include "acl.h"
48#include "truncate.h"
49
50#include <trace/events/ext4.h>
51
52static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
53			      struct ext4_inode_info *ei)
54{
55	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
56	__u32 csum;
57	__u16 dummy_csum = 0;
58	int offset = offsetof(struct ext4_inode, i_checksum_lo);
59	unsigned int csum_size = sizeof(dummy_csum);
60
61	csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
62	csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
63	offset += csum_size;
64	csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
65			   EXT4_GOOD_OLD_INODE_SIZE - offset);
66
67	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
68		offset = offsetof(struct ext4_inode, i_checksum_hi);
69		csum = ext4_chksum(sbi, csum, (__u8 *)raw +
70				   EXT4_GOOD_OLD_INODE_SIZE,
71				   offset - EXT4_GOOD_OLD_INODE_SIZE);
72		if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
73			csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
74					   csum_size);
75			offset += csum_size;
76		}
77		csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
78				   EXT4_INODE_SIZE(inode->i_sb) - offset);
79	}
80
81	return csum;
82}
83
84static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
85				  struct ext4_inode_info *ei)
86{
87	__u32 provided, calculated;
88
89	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
90	    cpu_to_le32(EXT4_OS_LINUX) ||
91	    !ext4_has_metadata_csum(inode->i_sb))
92		return 1;
93
94	provided = le16_to_cpu(raw->i_checksum_lo);
95	calculated = ext4_inode_csum(inode, raw, ei);
96	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
97	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
98		provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
99	else
100		calculated &= 0xFFFF;
101
102	return provided == calculated;
103}
104
105void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
106			 struct ext4_inode_info *ei)
107{
108	__u32 csum;
109
110	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
111	    cpu_to_le32(EXT4_OS_LINUX) ||
112	    !ext4_has_metadata_csum(inode->i_sb))
113		return;
114
115	csum = ext4_inode_csum(inode, raw, ei);
116	raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
117	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
118	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
119		raw->i_checksum_hi = cpu_to_le16(csum >> 16);
120}
121
122static inline int ext4_begin_ordered_truncate(struct inode *inode,
123					      loff_t new_size)
124{
125	trace_ext4_begin_ordered_truncate(inode, new_size);
126	/*
127	 * If jinode is zero, then we never opened the file for
128	 * writing, so there's no need to call
129	 * jbd2_journal_begin_ordered_truncate() since there's no
130	 * outstanding writes we need to flush.
131	 */
132	if (!EXT4_I(inode)->jinode)
133		return 0;
134	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
135						   EXT4_I(inode)->jinode,
136						   new_size);
137}
138
139static void ext4_invalidatepage(struct page *page, unsigned int offset,
140				unsigned int length);
141static int __ext4_journalled_writepage(struct page *page, unsigned int len);
142static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
143				  int pextents);
144
145/*
146 * Test whether an inode is a fast symlink.
147 * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
148 */
149int ext4_inode_is_fast_symlink(struct inode *inode)
150{
151	if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
152		int ea_blocks = EXT4_I(inode)->i_file_acl ?
153				EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
154
155		if (ext4_has_inline_data(inode))
156			return 0;
157
158		return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
159	}
160	return S_ISLNK(inode->i_mode) && inode->i_size &&
161	       (inode->i_size < EXT4_N_BLOCKS * 4);
162}
163
164/*
165 * Called at the last iput() if i_nlink is zero.
166 */
167void ext4_evict_inode(struct inode *inode)
168{
169	handle_t *handle;
170	int err;
171	/*
172	 * Credits for final inode cleanup and freeing:
173	 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
174	 * (xattr block freeing), bitmap, group descriptor (inode freeing)
175	 */
176	int extra_credits = 6;
177	struct ext4_xattr_inode_array *ea_inode_array = NULL;
178	bool freeze_protected = false;
179
180	trace_ext4_evict_inode(inode);
181
182	if (inode->i_nlink) {
183		/*
184		 * When journalling data dirty buffers are tracked only in the
185		 * journal. So although mm thinks everything is clean and
186		 * ready for reaping the inode might still have some pages to
187		 * write in the running transaction or waiting to be
188		 * checkpointed. Thus calling jbd2_journal_invalidatepage()
189		 * (via truncate_inode_pages()) to discard these buffers can
190		 * cause data loss. Also even if we did not discard these
191		 * buffers, we would have no way to find them after the inode
192		 * is reaped and thus user could see stale data if he tries to
193		 * read them before the transaction is checkpointed. So be
194		 * careful and force everything to disk here... We use
195		 * ei->i_datasync_tid to store the newest transaction
196		 * containing inode's data.
197		 *
198		 * Note that directories do not have this problem because they
199		 * don't use page cache.
200		 */
201		if (inode->i_ino != EXT4_JOURNAL_INO &&
202		    ext4_should_journal_data(inode) &&
203		    (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
204		    inode->i_data.nrpages) {
205			journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
206			tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
207
208			jbd2_complete_transaction(journal, commit_tid);
209			filemap_write_and_wait(&inode->i_data);
210		}
211		truncate_inode_pages_final(&inode->i_data);
212
213		goto no_delete;
214	}
215
216	if (is_bad_inode(inode))
217		goto no_delete;
218	dquot_initialize(inode);
219
220	if (ext4_should_order_data(inode))
221		ext4_begin_ordered_truncate(inode, 0);
222	truncate_inode_pages_final(&inode->i_data);
223
224	/*
225	 * For inodes with journalled data, transaction commit could have
226	 * dirtied the inode. Flush worker is ignoring it because of I_FREEING
227	 * flag but we still need to remove the inode from the writeback lists.
228	 */
229	if (!list_empty_careful(&inode->i_io_list)) {
230		WARN_ON_ONCE(!ext4_should_journal_data(inode));
231		inode_io_list_del(inode);
232	}
233
234	/*
235	 * Protect us against freezing - iput() caller didn't have to have any
236	 * protection against it. When we are in a running transaction though,
237	 * we are already protected against freezing and we cannot grab further
238	 * protection due to lock ordering constraints.
239	 */
240	if (!ext4_journal_current_handle()) {
241		sb_start_intwrite(inode->i_sb);
242		freeze_protected = true;
243	}
244
245	if (!IS_NOQUOTA(inode))
246		extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
247
248	/*
249	 * Block bitmap, group descriptor, and inode are accounted in both
250	 * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
251	 */
252	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
253			 ext4_blocks_for_truncate(inode) + extra_credits - 3);
254	if (IS_ERR(handle)) {
255		ext4_std_error(inode->i_sb, PTR_ERR(handle));
256		/*
257		 * If we're going to skip the normal cleanup, we still need to
258		 * make sure that the in-core orphan linked list is properly
259		 * cleaned up.
260		 */
261		ext4_orphan_del(NULL, inode);
262		if (freeze_protected)
263			sb_end_intwrite(inode->i_sb);
264		goto no_delete;
265	}
266
267	if (IS_SYNC(inode))
268		ext4_handle_sync(handle);
269
270	/*
271	 * Set inode->i_size to 0 before calling ext4_truncate(). We need
272	 * special handling of symlinks here because i_size is used to
273	 * determine whether ext4_inode_info->i_data contains symlink data or
274	 * block mappings. Setting i_size to 0 will remove its fast symlink
275	 * status. Erase i_data so that it becomes a valid empty block map.
276	 */
277	if (ext4_inode_is_fast_symlink(inode))
278		memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
279	inode->i_size = 0;
280	err = ext4_mark_inode_dirty(handle, inode);
281	if (err) {
282		ext4_warning(inode->i_sb,
283			     "couldn't mark inode dirty (err %d)", err);
284		goto stop_handle;
285	}
286	if (inode->i_blocks) {
287		err = ext4_truncate(inode);
288		if (err) {
289			ext4_error_err(inode->i_sb, -err,
290				       "couldn't truncate inode %lu (err %d)",
291				       inode->i_ino, err);
292			goto stop_handle;
293		}
294	}
295
296	/* Remove xattr references. */
297	err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
298				      extra_credits);
299	if (err) {
300		ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
301stop_handle:
302		ext4_journal_stop(handle);
303		ext4_orphan_del(NULL, inode);
304		if (freeze_protected)
305			sb_end_intwrite(inode->i_sb);
306		ext4_xattr_inode_array_free(ea_inode_array);
307		goto no_delete;
308	}
309
310	/*
311	 * Kill off the orphan record which ext4_truncate created.
312	 * AKPM: I think this can be inside the above `if'.
313	 * Note that ext4_orphan_del() has to be able to cope with the
314	 * deletion of a non-existent orphan - this is because we don't
315	 * know if ext4_truncate() actually created an orphan record.
316	 * (Well, we could do this if we need to, but heck - it works)
317	 */
318	ext4_orphan_del(handle, inode);
319	EXT4_I(inode)->i_dtime	= (__u32)ktime_get_real_seconds();
320
321	/*
322	 * One subtle ordering requirement: if anything has gone wrong
323	 * (transaction abort, IO errors, whatever), then we can still
324	 * do these next steps (the fs will already have been marked as
325	 * having errors), but we can't free the inode if the mark_dirty
326	 * fails.
327	 */
328	if (ext4_mark_inode_dirty(handle, inode))
329		/* If that failed, just do the required in-core inode clear. */
330		ext4_clear_inode(inode);
331	else
332		ext4_free_inode(handle, inode);
333	ext4_journal_stop(handle);
334	if (freeze_protected)
335		sb_end_intwrite(inode->i_sb);
336	ext4_xattr_inode_array_free(ea_inode_array);
337	return;
338no_delete:
339	if (!list_empty(&EXT4_I(inode)->i_fc_list))
340		ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM);
341	ext4_clear_inode(inode);	/* We must guarantee clearing of inode... */
342}
343
344#ifdef CONFIG_QUOTA
345qsize_t *ext4_get_reserved_space(struct inode *inode)
346{
347	return &EXT4_I(inode)->i_reserved_quota;
348}
349#endif
350
351/*
352 * Called with i_data_sem down, which is important since we can call
353 * ext4_discard_preallocations() from here.
354 */
355void ext4_da_update_reserve_space(struct inode *inode,
356					int used, int quota_claim)
357{
358	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
359	struct ext4_inode_info *ei = EXT4_I(inode);
360
361	spin_lock(&ei->i_block_reservation_lock);
362	trace_ext4_da_update_reserve_space(inode, used, quota_claim);
363	if (unlikely(used > ei->i_reserved_data_blocks)) {
364		ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
365			 "with only %d reserved data blocks",
366			 __func__, inode->i_ino, used,
367			 ei->i_reserved_data_blocks);
368		WARN_ON(1);
369		used = ei->i_reserved_data_blocks;
370	}
371
372	/* Update per-inode reservations */
373	ei->i_reserved_data_blocks -= used;
374	percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
375
376	spin_unlock(&ei->i_block_reservation_lock);
377
378	/* Update quota subsystem for data blocks */
379	if (quota_claim)
380		dquot_claim_block(inode, EXT4_C2B(sbi, used));
381	else {
382		/*
383		 * We did fallocate with an offset that is already delayed
384		 * allocated. So on delayed allocated writeback we should
385		 * not re-claim the quota for fallocated blocks.
386		 */
387		dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
388	}
389
390	/*
391	 * If we have done all the pending block allocations and if
392	 * there aren't any writers on the inode, we can discard the
393	 * inode's preallocations.
394	 */
395	if ((ei->i_reserved_data_blocks == 0) &&
396	    !inode_is_open_for_write(inode))
397		ext4_discard_preallocations(inode, 0);
398}
399
400static int __check_block_validity(struct inode *inode, const char *func,
401				unsigned int line,
402				struct ext4_map_blocks *map)
403{
404	if (ext4_has_feature_journal(inode->i_sb) &&
405	    (inode->i_ino ==
406	     le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
407		return 0;
408	if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
409		ext4_error_inode(inode, func, line, map->m_pblk,
410				 "lblock %lu mapped to illegal pblock %llu "
411				 "(length %d)", (unsigned long) map->m_lblk,
412				 map->m_pblk, map->m_len);
413		return -EFSCORRUPTED;
414	}
415	return 0;
416}
417
418int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
419		       ext4_lblk_t len)
420{
421	int ret;
422
423	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
424		return fscrypt_zeroout_range(inode, lblk, pblk, len);
425
426	ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
427	if (ret > 0)
428		ret = 0;
429
430	return ret;
431}
432
433#define check_block_validity(inode, map)	\
434	__check_block_validity((inode), __func__, __LINE__, (map))
435
436#ifdef ES_AGGRESSIVE_TEST
437static void ext4_map_blocks_es_recheck(handle_t *handle,
438				       struct inode *inode,
439				       struct ext4_map_blocks *es_map,
440				       struct ext4_map_blocks *map,
441				       int flags)
442{
443	int retval;
444
445	map->m_flags = 0;
446	/*
447	 * There is a race window that the result is not the same.
448	 * e.g. xfstests #223 when dioread_nolock enables.  The reason
449	 * is that we lookup a block mapping in extent status tree with
450	 * out taking i_data_sem.  So at the time the unwritten extent
451	 * could be converted.
452	 */
453	down_read(&EXT4_I(inode)->i_data_sem);
454	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
455		retval = ext4_ext_map_blocks(handle, inode, map, 0);
456	} else {
457		retval = ext4_ind_map_blocks(handle, inode, map, 0);
458	}
459	up_read((&EXT4_I(inode)->i_data_sem));
460
461	/*
462	 * We don't check m_len because extent will be collpased in status
463	 * tree.  So the m_len might not equal.
464	 */
465	if (es_map->m_lblk != map->m_lblk ||
466	    es_map->m_flags != map->m_flags ||
467	    es_map->m_pblk != map->m_pblk) {
468		printk("ES cache assertion failed for inode: %lu "
469		       "es_cached ex [%d/%d/%llu/%x] != "
470		       "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
471		       inode->i_ino, es_map->m_lblk, es_map->m_len,
472		       es_map->m_pblk, es_map->m_flags, map->m_lblk,
473		       map->m_len, map->m_pblk, map->m_flags,
474		       retval, flags);
475	}
476}
477#endif /* ES_AGGRESSIVE_TEST */
478
479/*
480 * The ext4_map_blocks() function tries to look up the requested blocks,
481 * and returns if the blocks are already mapped.
482 *
483 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
484 * and store the allocated blocks in the result buffer head and mark it
485 * mapped.
486 *
487 * If file type is extents based, it will call ext4_ext_map_blocks(),
488 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
489 * based files
490 *
491 * On success, it returns the number of blocks being mapped or allocated.  if
492 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
493 * is marked as unwritten. If the create == 1, it will mark @map as mapped.
494 *
495 * It returns 0 if plain look up failed (blocks have not been allocated), in
496 * that case, @map is returned as unmapped but we still do fill map->m_len to
497 * indicate the length of a hole starting at map->m_lblk.
498 *
499 * It returns the error in case of allocation failure.
500 */
501int ext4_map_blocks(handle_t *handle, struct inode *inode,
502		    struct ext4_map_blocks *map, int flags)
503{
504	struct extent_status es;
505	int retval;
506	int ret = 0;
507#ifdef ES_AGGRESSIVE_TEST
508	struct ext4_map_blocks orig_map;
509
510	memcpy(&orig_map, map, sizeof(*map));
511#endif
512
513	map->m_flags = 0;
514	ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
515		  flags, map->m_len, (unsigned long) map->m_lblk);
516
517	/*
518	 * ext4_map_blocks returns an int, and m_len is an unsigned int
519	 */
520	if (unlikely(map->m_len > INT_MAX))
521		map->m_len = INT_MAX;
522
523	/* We can handle the block number less than EXT_MAX_BLOCKS */
524	if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
525		return -EFSCORRUPTED;
526
527	/* Lookup extent status tree firstly */
528	if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&
529	    ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
530		if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
531			map->m_pblk = ext4_es_pblock(&es) +
532					map->m_lblk - es.es_lblk;
533			map->m_flags |= ext4_es_is_written(&es) ?
534					EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
535			retval = es.es_len - (map->m_lblk - es.es_lblk);
536			if (retval > map->m_len)
537				retval = map->m_len;
538			map->m_len = retval;
539		} else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
540			map->m_pblk = 0;
541			retval = es.es_len - (map->m_lblk - es.es_lblk);
542			if (retval > map->m_len)
543				retval = map->m_len;
544			map->m_len = retval;
545			retval = 0;
546		} else {
547			BUG();
548		}
549#ifdef ES_AGGRESSIVE_TEST
550		ext4_map_blocks_es_recheck(handle, inode, map,
551					   &orig_map, flags);
552#endif
553		goto found;
554	}
555
556	/*
557	 * Try to see if we can get the block without requesting a new
558	 * file system block.
559	 */
560	down_read(&EXT4_I(inode)->i_data_sem);
561	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
562		retval = ext4_ext_map_blocks(handle, inode, map, 0);
563	} else {
564		retval = ext4_ind_map_blocks(handle, inode, map, 0);
565	}
566	if (retval > 0) {
567		unsigned int status;
568
569		if (unlikely(retval != map->m_len)) {
570			ext4_warning(inode->i_sb,
571				     "ES len assertion failed for inode "
572				     "%lu: retval %d != map->m_len %d",
573				     inode->i_ino, retval, map->m_len);
574			WARN_ON(1);
575		}
576
577		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
578				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
579		if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
580		    !(status & EXTENT_STATUS_WRITTEN) &&
581		    ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
582				       map->m_lblk + map->m_len - 1))
583			status |= EXTENT_STATUS_DELAYED;
584		ret = ext4_es_insert_extent(inode, map->m_lblk,
585					    map->m_len, map->m_pblk, status);
586		if (ret < 0)
587			retval = ret;
588	}
589	up_read((&EXT4_I(inode)->i_data_sem));
590
591found:
592	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
593		ret = check_block_validity(inode, map);
594		if (ret != 0)
595			return ret;
596	}
597
598	/* If it is only a block(s) look up */
599	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
600		return retval;
601
602	/*
603	 * Returns if the blocks have already allocated
604	 *
605	 * Note that if blocks have been preallocated
606	 * ext4_ext_get_block() returns the create = 0
607	 * with buffer head unmapped.
608	 */
609	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
610		/*
611		 * If we need to convert extent to unwritten
612		 * we continue and do the actual work in
613		 * ext4_ext_map_blocks()
614		 */
615		if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
616			return retval;
617
618	/*
619	 * Here we clear m_flags because after allocating an new extent,
620	 * it will be set again.
621	 */
622	map->m_flags &= ~EXT4_MAP_FLAGS;
623
624	/*
625	 * New blocks allocate and/or writing to unwritten extent
626	 * will possibly result in updating i_data, so we take
627	 * the write lock of i_data_sem, and call get_block()
628	 * with create == 1 flag.
629	 */
630	down_write(&EXT4_I(inode)->i_data_sem);
631
632	/*
633	 * We need to check for EXT4 here because migrate
634	 * could have changed the inode type in between
635	 */
636	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
637		retval = ext4_ext_map_blocks(handle, inode, map, flags);
638	} else {
639		retval = ext4_ind_map_blocks(handle, inode, map, flags);
640
641		if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
642			/*
643			 * We allocated new blocks which will result in
644			 * i_data's format changing.  Force the migrate
645			 * to fail by clearing migrate flags
646			 */
647			ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
648		}
649
650		/*
651		 * Update reserved blocks/metadata blocks after successful
652		 * block allocation which had been deferred till now. We don't
653		 * support fallocate for non extent files. So we can update
654		 * reserve space here.
655		 */
656		if ((retval > 0) &&
657			(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
658			ext4_da_update_reserve_space(inode, retval, 1);
659	}
660
661	if (retval > 0) {
662		unsigned int status;
663
664		if (unlikely(retval != map->m_len)) {
665			ext4_warning(inode->i_sb,
666				     "ES len assertion failed for inode "
667				     "%lu: retval %d != map->m_len %d",
668				     inode->i_ino, retval, map->m_len);
669			WARN_ON(1);
670		}
671
672		/*
673		 * We have to zeroout blocks before inserting them into extent
674		 * status tree. Otherwise someone could look them up there and
675		 * use them before they are really zeroed. We also have to
676		 * unmap metadata before zeroing as otherwise writeback can
677		 * overwrite zeros with stale data from block device.
678		 */
679		if (flags & EXT4_GET_BLOCKS_ZERO &&
680		    map->m_flags & EXT4_MAP_MAPPED &&
681		    map->m_flags & EXT4_MAP_NEW) {
682			ret = ext4_issue_zeroout(inode, map->m_lblk,
683						 map->m_pblk, map->m_len);
684			if (ret) {
685				retval = ret;
686				goto out_sem;
687			}
688		}
689
690		/*
691		 * If the extent has been zeroed out, we don't need to update
692		 * extent status tree.
693		 */
694		if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
695		    ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
696			if (ext4_es_is_written(&es))
697				goto out_sem;
698		}
699		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
700				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
701		if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
702		    !(status & EXTENT_STATUS_WRITTEN) &&
703		    ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
704				       map->m_lblk + map->m_len - 1))
705			status |= EXTENT_STATUS_DELAYED;
706		ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
707					    map->m_pblk, status);
708		if (ret < 0) {
709			retval = ret;
710			goto out_sem;
711		}
712	}
713
714out_sem:
715	up_write((&EXT4_I(inode)->i_data_sem));
716	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
717		ret = check_block_validity(inode, map);
718		if (ret != 0)
719			return ret;
720
721		/*
722		 * Inodes with freshly allocated blocks where contents will be
723		 * visible after transaction commit must be on transaction's
724		 * ordered data list.
725		 */
726		if (map->m_flags & EXT4_MAP_NEW &&
727		    !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
728		    !(flags & EXT4_GET_BLOCKS_ZERO) &&
729		    !ext4_is_quota_file(inode) &&
730		    ext4_should_order_data(inode)) {
731			loff_t start_byte =
732				(loff_t)map->m_lblk << inode->i_blkbits;
733			loff_t length = (loff_t)map->m_len << inode->i_blkbits;
734
735			if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
736				ret = ext4_jbd2_inode_add_wait(handle, inode,
737						start_byte, length);
738			else
739				ret = ext4_jbd2_inode_add_write(handle, inode,
740						start_byte, length);
741			if (ret)
742				return ret;
743		}
744		ext4_fc_track_range(handle, inode, map->m_lblk,
745			    map->m_lblk + map->m_len - 1);
746	}
747
748	if (retval < 0)
749		ext_debug(inode, "failed with err %d\n", retval);
750	return retval;
751}
752
753/*
754 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
755 * we have to be careful as someone else may be manipulating b_state as well.
756 */
757static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
758{
759	unsigned long old_state;
760	unsigned long new_state;
761
762	flags &= EXT4_MAP_FLAGS;
763
764	/* Dummy buffer_head? Set non-atomically. */
765	if (!bh->b_page) {
766		bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
767		return;
768	}
769	/*
770	 * Someone else may be modifying b_state. Be careful! This is ugly but
771	 * once we get rid of using bh as a container for mapping information
772	 * to pass to / from get_block functions, this can go away.
773	 */
774	do {
775		old_state = READ_ONCE(bh->b_state);
776		new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
777	} while (unlikely(
778		 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
779}
780
781static int _ext4_get_block(struct inode *inode, sector_t iblock,
782			   struct buffer_head *bh, int flags)
783{
784	struct ext4_map_blocks map;
785	int ret = 0;
786
787	if (ext4_has_inline_data(inode))
788		return -ERANGE;
789
790	map.m_lblk = iblock;
791	map.m_len = bh->b_size >> inode->i_blkbits;
792
793	ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
794			      flags);
795	if (ret > 0) {
796		map_bh(bh, inode->i_sb, map.m_pblk);
797		ext4_update_bh_state(bh, map.m_flags);
798		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
799		ret = 0;
800	} else if (ret == 0) {
801		/* hole case, need to fill in bh->b_size */
802		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
803	}
804	return ret;
805}
806
807int ext4_get_block(struct inode *inode, sector_t iblock,
808		   struct buffer_head *bh, int create)
809{
810	return _ext4_get_block(inode, iblock, bh,
811			       create ? EXT4_GET_BLOCKS_CREATE : 0);
812}
813
814/*
815 * Get block function used when preparing for buffered write if we require
816 * creating an unwritten extent if blocks haven't been allocated.  The extent
817 * will be converted to written after the IO is complete.
818 */
819int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
820			     struct buffer_head *bh_result, int create)
821{
822	ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
823		   inode->i_ino, create);
824	return _ext4_get_block(inode, iblock, bh_result,
825			       EXT4_GET_BLOCKS_IO_CREATE_EXT);
826}
827
828/* Maximum number of blocks we map for direct IO at once. */
829#define DIO_MAX_BLOCKS 4096
830
831/*
832 * `handle' can be NULL if create is zero
833 */
834struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
835				ext4_lblk_t block, int map_flags)
836{
837	struct ext4_map_blocks map;
838	struct buffer_head *bh;
839	int create = map_flags & EXT4_GET_BLOCKS_CREATE;
840	int err;
841
842	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
843		    || handle != NULL || create == 0);
844
845	map.m_lblk = block;
846	map.m_len = 1;
847	err = ext4_map_blocks(handle, inode, &map, map_flags);
848
849	if (err == 0)
850		return create ? ERR_PTR(-ENOSPC) : NULL;
851	if (err < 0)
852		return ERR_PTR(err);
853
854	bh = sb_getblk(inode->i_sb, map.m_pblk);
855	if (unlikely(!bh))
856		return ERR_PTR(-ENOMEM);
857	if (map.m_flags & EXT4_MAP_NEW) {
858		ASSERT(create != 0);
859		ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
860			    || (handle != NULL));
861
862		/*
863		 * Now that we do not always journal data, we should
864		 * keep in mind whether this should always journal the
865		 * new buffer as metadata.  For now, regular file
866		 * writes use ext4_get_block instead, so it's not a
867		 * problem.
868		 */
869		lock_buffer(bh);
870		BUFFER_TRACE(bh, "call get_create_access");
871		err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
872						     EXT4_JTR_NONE);
873		if (unlikely(err)) {
874			unlock_buffer(bh);
875			goto errout;
876		}
877		if (!buffer_uptodate(bh)) {
878			memset(bh->b_data, 0, inode->i_sb->s_blocksize);
879			set_buffer_uptodate(bh);
880		}
881		unlock_buffer(bh);
882		BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
883		err = ext4_handle_dirty_metadata(handle, inode, bh);
884		if (unlikely(err))
885			goto errout;
886	} else
887		BUFFER_TRACE(bh, "not a new buffer");
888	return bh;
889errout:
890	brelse(bh);
891	return ERR_PTR(err);
892}
893
894struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
895			       ext4_lblk_t block, int map_flags)
896{
897	struct buffer_head *bh;
898	int ret;
899
900	bh = ext4_getblk(handle, inode, block, map_flags);
901	if (IS_ERR(bh))
902		return bh;
903	if (!bh || ext4_buffer_uptodate(bh))
904		return bh;
905
906	ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
907	if (ret) {
908		put_bh(bh);
909		return ERR_PTR(ret);
910	}
911	return bh;
912}
913
914/* Read a contiguous batch of blocks. */
915int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
916		     bool wait, struct buffer_head **bhs)
917{
918	int i, err;
919
920	for (i = 0; i < bh_count; i++) {
921		bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
922		if (IS_ERR(bhs[i])) {
923			err = PTR_ERR(bhs[i]);
924			bh_count = i;
925			goto out_brelse;
926		}
927	}
928
929	for (i = 0; i < bh_count; i++)
930		/* Note that NULL bhs[i] is valid because of holes. */
931		if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
932			ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
933
934	if (!wait)
935		return 0;
936
937	for (i = 0; i < bh_count; i++)
938		if (bhs[i])
939			wait_on_buffer(bhs[i]);
940
941	for (i = 0; i < bh_count; i++) {
942		if (bhs[i] && !buffer_uptodate(bhs[i])) {
943			err = -EIO;
944			goto out_brelse;
945		}
946	}
947	return 0;
948
949out_brelse:
950	for (i = 0; i < bh_count; i++) {
951		brelse(bhs[i]);
952		bhs[i] = NULL;
953	}
954	return err;
955}
956
957int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
958			   struct buffer_head *head,
959			   unsigned from,
960			   unsigned to,
961			   int *partial,
962			   int (*fn)(handle_t *handle, struct inode *inode,
963				     struct buffer_head *bh))
964{
965	struct buffer_head *bh;
966	unsigned block_start, block_end;
967	unsigned blocksize = head->b_size;
968	int err, ret = 0;
969	struct buffer_head *next;
970
971	for (bh = head, block_start = 0;
972	     ret == 0 && (bh != head || !block_start);
973	     block_start = block_end, bh = next) {
974		next = bh->b_this_page;
975		block_end = block_start + blocksize;
976		if (block_end <= from || block_start >= to) {
977			if (partial && !buffer_uptodate(bh))
978				*partial = 1;
979			continue;
980		}
981		err = (*fn)(handle, inode, bh);
982		if (!ret)
983			ret = err;
984	}
985	return ret;
986}
987
988/*
989 * To preserve ordering, it is essential that the hole instantiation and
990 * the data write be encapsulated in a single transaction.  We cannot
991 * close off a transaction and start a new one between the ext4_get_block()
992 * and the commit_write().  So doing the jbd2_journal_start at the start of
993 * prepare_write() is the right place.
994 *
995 * Also, this function can nest inside ext4_writepage().  In that case, we
996 * *know* that ext4_writepage() has generated enough buffer credits to do the
997 * whole page.  So we won't block on the journal in that case, which is good,
998 * because the caller may be PF_MEMALLOC.
999 *
1000 * By accident, ext4 can be reentered when a transaction is open via
1001 * quota file writes.  If we were to commit the transaction while thus
1002 * reentered, there can be a deadlock - we would be holding a quota
1003 * lock, and the commit would never complete if another thread had a
1004 * transaction open and was blocking on the quota lock - a ranking
1005 * violation.
1006 *
1007 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
1008 * will _not_ run commit under these circumstances because handle->h_ref
1009 * is elevated.  We'll still have enough credits for the tiny quotafile
1010 * write.
1011 */
1012int do_journal_get_write_access(handle_t *handle, struct inode *inode,
1013				struct buffer_head *bh)
1014{
1015	int dirty = buffer_dirty(bh);
1016	int ret;
1017
1018	if (!buffer_mapped(bh) || buffer_freed(bh))
1019		return 0;
1020	/*
1021	 * __block_write_begin() could have dirtied some buffers. Clean
1022	 * the dirty bit as jbd2_journal_get_write_access() could complain
1023	 * otherwise about fs integrity issues. Setting of the dirty bit
1024	 * by __block_write_begin() isn't a real problem here as we clear
1025	 * the bit before releasing a page lock and thus writeback cannot
1026	 * ever write the buffer.
1027	 */
1028	if (dirty)
1029		clear_buffer_dirty(bh);
1030	BUFFER_TRACE(bh, "get write access");
1031	ret = ext4_journal_get_write_access(handle, inode->i_sb, bh,
1032					    EXT4_JTR_NONE);
1033	if (!ret && dirty)
1034		ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1035	return ret;
1036}
1037
1038#ifdef CONFIG_FS_ENCRYPTION
1039static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1040				  get_block_t *get_block)
1041{
1042	unsigned from = pos & (PAGE_SIZE - 1);
1043	unsigned to = from + len;
1044	struct inode *inode = page->mapping->host;
1045	unsigned block_start, block_end;
1046	sector_t block;
1047	int err = 0;
1048	unsigned blocksize = inode->i_sb->s_blocksize;
1049	unsigned bbits;
1050	struct buffer_head *bh, *head, *wait[2];
1051	int nr_wait = 0;
1052	int i;
1053
1054	BUG_ON(!PageLocked(page));
1055	BUG_ON(from > PAGE_SIZE);
1056	BUG_ON(to > PAGE_SIZE);
1057	BUG_ON(from > to);
1058
1059	if (!page_has_buffers(page))
1060		create_empty_buffers(page, blocksize, 0);
1061	head = page_buffers(page);
1062	bbits = ilog2(blocksize);
1063	block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1064
1065	for (bh = head, block_start = 0; bh != head || !block_start;
1066	    block++, block_start = block_end, bh = bh->b_this_page) {
1067		block_end = block_start + blocksize;
1068		if (block_end <= from || block_start >= to) {
1069			if (PageUptodate(page)) {
1070				set_buffer_uptodate(bh);
1071			}
1072			continue;
1073		}
1074		if (buffer_new(bh))
1075			clear_buffer_new(bh);
1076		if (!buffer_mapped(bh)) {
1077			WARN_ON(bh->b_size != blocksize);
1078			err = get_block(inode, block, bh, 1);
1079			if (err)
1080				break;
1081			if (buffer_new(bh)) {
1082				if (PageUptodate(page)) {
1083					clear_buffer_new(bh);
1084					set_buffer_uptodate(bh);
1085					mark_buffer_dirty(bh);
1086					continue;
1087				}
1088				if (block_end > to || block_start < from)
1089					zero_user_segments(page, to, block_end,
1090							   block_start, from);
1091				continue;
1092			}
1093		}
1094		if (PageUptodate(page)) {
1095			set_buffer_uptodate(bh);
1096			continue;
1097		}
1098		if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1099		    !buffer_unwritten(bh) &&
1100		    (block_start < from || block_end > to)) {
1101			ext4_read_bh_lock(bh, 0, false);
1102			wait[nr_wait++] = bh;
1103		}
1104	}
1105	/*
1106	 * If we issued read requests, let them complete.
1107	 */
1108	for (i = 0; i < nr_wait; i++) {
1109		wait_on_buffer(wait[i]);
1110		if (!buffer_uptodate(wait[i]))
1111			err = -EIO;
1112	}
1113	if (unlikely(err)) {
1114		page_zero_new_buffers(page, from, to);
1115	} else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
1116		for (i = 0; i < nr_wait; i++) {
1117			int err2;
1118
1119			err2 = fscrypt_decrypt_pagecache_blocks(page, blocksize,
1120								bh_offset(wait[i]));
1121			if (err2) {
1122				clear_buffer_uptodate(wait[i]);
1123				err = err2;
1124			}
1125		}
1126	}
1127
1128	return err;
1129}
1130#endif
1131
1132static int ext4_write_begin(struct file *file, struct address_space *mapping,
1133			    loff_t pos, unsigned len, unsigned flags,
1134			    struct page **pagep, void **fsdata)
1135{
1136	struct inode *inode = mapping->host;
1137	int ret, needed_blocks;
1138	handle_t *handle;
1139	int retries = 0;
1140	struct page *page;
1141	pgoff_t index;
1142	unsigned from, to;
1143
1144	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1145		return -EIO;
1146
1147	trace_ext4_write_begin(inode, pos, len, flags);
1148	/*
1149	 * Reserve one block more for addition to orphan list in case
1150	 * we allocate blocks but write fails for some reason
1151	 */
1152	needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1153	index = pos >> PAGE_SHIFT;
1154	from = pos & (PAGE_SIZE - 1);
1155	to = from + len;
1156
1157	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1158		ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1159						    flags, pagep);
1160		if (ret < 0)
1161			return ret;
1162		if (ret == 1)
1163			return 0;
1164	}
1165
1166	/*
1167	 * grab_cache_page_write_begin() can take a long time if the
1168	 * system is thrashing due to memory pressure, or if the page
1169	 * is being written back.  So grab it first before we start
1170	 * the transaction handle.  This also allows us to allocate
1171	 * the page (if needed) without using GFP_NOFS.
1172	 */
1173retry_grab:
1174	page = grab_cache_page_write_begin(mapping, index, flags);
1175	if (!page)
1176		return -ENOMEM;
1177	unlock_page(page);
1178
1179retry_journal:
1180	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1181	if (IS_ERR(handle)) {
1182		put_page(page);
1183		return PTR_ERR(handle);
1184	}
1185
1186	lock_page(page);
1187	if (page->mapping != mapping) {
1188		/* The page got truncated from under us */
1189		unlock_page(page);
1190		put_page(page);
1191		ext4_journal_stop(handle);
1192		goto retry_grab;
1193	}
1194	/* In case writeback began while the page was unlocked */
1195	wait_for_stable_page(page);
1196
1197#ifdef CONFIG_FS_ENCRYPTION
1198	if (ext4_should_dioread_nolock(inode))
1199		ret = ext4_block_write_begin(page, pos, len,
1200					     ext4_get_block_unwritten);
1201	else
1202		ret = ext4_block_write_begin(page, pos, len,
1203					     ext4_get_block);
1204#else
1205	if (ext4_should_dioread_nolock(inode))
1206		ret = __block_write_begin(page, pos, len,
1207					  ext4_get_block_unwritten);
1208	else
1209		ret = __block_write_begin(page, pos, len, ext4_get_block);
1210#endif
1211	if (!ret && ext4_should_journal_data(inode)) {
1212		ret = ext4_walk_page_buffers(handle, inode,
1213					     page_buffers(page), from, to, NULL,
1214					     do_journal_get_write_access);
1215	}
1216
1217	if (ret) {
1218		bool extended = (pos + len > inode->i_size) &&
1219				!ext4_verity_in_progress(inode);
1220
1221		unlock_page(page);
1222		/*
1223		 * __block_write_begin may have instantiated a few blocks
1224		 * outside i_size.  Trim these off again. Don't need
1225		 * i_size_read because we hold i_mutex.
1226		 *
1227		 * Add inode to orphan list in case we crash before
1228		 * truncate finishes
1229		 */
1230		if (extended && ext4_can_truncate(inode))
1231			ext4_orphan_add(handle, inode);
1232
1233		ext4_journal_stop(handle);
1234		if (extended) {
1235			ext4_truncate_failed_write(inode);
1236			/*
1237			 * If truncate failed early the inode might
1238			 * still be on the orphan list; we need to
1239			 * make sure the inode is removed from the
1240			 * orphan list in that case.
1241			 */
1242			if (inode->i_nlink)
1243				ext4_orphan_del(NULL, inode);
1244		}
1245
1246		if (ret == -ENOSPC &&
1247		    ext4_should_retry_alloc(inode->i_sb, &retries))
1248			goto retry_journal;
1249		put_page(page);
1250		return ret;
1251	}
1252	*pagep = page;
1253	return ret;
1254}
1255
1256/* For write_end() in data=journal mode */
1257static int write_end_fn(handle_t *handle, struct inode *inode,
1258			struct buffer_head *bh)
1259{
1260	int ret;
1261	if (!buffer_mapped(bh) || buffer_freed(bh))
1262		return 0;
1263	set_buffer_uptodate(bh);
1264	ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1265	clear_buffer_meta(bh);
1266	clear_buffer_prio(bh);
1267	return ret;
1268}
1269
1270/*
1271 * We need to pick up the new inode size which generic_commit_write gave us
1272 * `file' can be NULL - eg, when called from page_symlink().
1273 *
1274 * ext4 never places buffers on inode->i_mapping->private_list.  metadata
1275 * buffers are managed internally.
1276 */
1277static int ext4_write_end(struct file *file,
1278			  struct address_space *mapping,
1279			  loff_t pos, unsigned len, unsigned copied,
1280			  struct page *page, void *fsdata)
1281{
1282	handle_t *handle = ext4_journal_current_handle();
1283	struct inode *inode = mapping->host;
1284	loff_t old_size = inode->i_size;
1285	int ret = 0, ret2;
1286	int i_size_changed = 0;
1287	int inline_data = ext4_has_inline_data(inode);
1288	bool verity = ext4_verity_in_progress(inode);
1289
1290	trace_ext4_write_end(inode, pos, len, copied);
1291	if (inline_data) {
1292		ret = ext4_write_inline_data_end(inode, pos, len,
1293						 copied, page);
1294		if (ret < 0) {
1295			unlock_page(page);
1296			put_page(page);
1297			goto errout;
1298		}
1299		copied = ret;
1300	} else
1301		copied = block_write_end(file, mapping, pos,
1302					 len, copied, page, fsdata);
1303	/*
1304	 * it's important to update i_size while still holding page lock:
1305	 * page writeout could otherwise come in and zero beyond i_size.
1306	 *
1307	 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1308	 * blocks are being written past EOF, so skip the i_size update.
1309	 */
1310	if (!verity)
1311		i_size_changed = ext4_update_inode_size(inode, pos + copied);
1312	unlock_page(page);
1313	put_page(page);
1314
1315	if (old_size < pos && !verity)
1316		pagecache_isize_extended(inode, old_size, pos);
1317	/*
1318	 * Don't mark the inode dirty under page lock. First, it unnecessarily
1319	 * makes the holding time of page lock longer. Second, it forces lock
1320	 * ordering of page lock and transaction start for journaling
1321	 * filesystems.
1322	 */
1323	if (i_size_changed || inline_data)
1324		ret = ext4_mark_inode_dirty(handle, inode);
1325
1326	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1327		/* if we have allocated more blocks and copied
1328		 * less. We will have blocks allocated outside
1329		 * inode->i_size. So truncate them
1330		 */
1331		ext4_orphan_add(handle, inode);
1332errout:
1333	ret2 = ext4_journal_stop(handle);
1334	if (!ret)
1335		ret = ret2;
1336
1337	if (pos + len > inode->i_size && !verity) {
1338		ext4_truncate_failed_write(inode);
1339		/*
1340		 * If truncate failed early the inode might still be
1341		 * on the orphan list; we need to make sure the inode
1342		 * is removed from the orphan list in that case.
1343		 */
1344		if (inode->i_nlink)
1345			ext4_orphan_del(NULL, inode);
1346	}
1347
1348	return ret ? ret : copied;
1349}
1350
1351/*
1352 * This is a private version of page_zero_new_buffers() which doesn't
1353 * set the buffer to be dirty, since in data=journalled mode we need
1354 * to call ext4_handle_dirty_metadata() instead.
1355 */
1356static void ext4_journalled_zero_new_buffers(handle_t *handle,
1357					    struct inode *inode,
1358					    struct page *page,
1359					    unsigned from, unsigned to)
1360{
1361	unsigned int block_start = 0, block_end;
1362	struct buffer_head *head, *bh;
1363
1364	bh = head = page_buffers(page);
1365	do {
1366		block_end = block_start + bh->b_size;
1367		if (buffer_new(bh)) {
1368			if (block_end > from && block_start < to) {
1369				if (!PageUptodate(page)) {
1370					unsigned start, size;
1371
1372					start = max(from, block_start);
1373					size = min(to, block_end) - start;
1374
1375					zero_user(page, start, size);
1376					write_end_fn(handle, inode, bh);
1377				}
1378				clear_buffer_new(bh);
1379			}
1380		}
1381		block_start = block_end;
1382		bh = bh->b_this_page;
1383	} while (bh != head);
1384}
1385
1386static int ext4_journalled_write_end(struct file *file,
1387				     struct address_space *mapping,
1388				     loff_t pos, unsigned len, unsigned copied,
1389				     struct page *page, void *fsdata)
1390{
1391	handle_t *handle = ext4_journal_current_handle();
1392	struct inode *inode = mapping->host;
1393	loff_t old_size = inode->i_size;
1394	int ret = 0, ret2;
1395	int partial = 0;
1396	unsigned from, to;
1397	int size_changed = 0;
1398	int inline_data = ext4_has_inline_data(inode);
1399	bool verity = ext4_verity_in_progress(inode);
1400
1401	trace_ext4_journalled_write_end(inode, pos, len, copied);
1402	from = pos & (PAGE_SIZE - 1);
1403	to = from + len;
1404
1405	BUG_ON(!ext4_handle_valid(handle));
1406
1407	if (inline_data) {
1408		ret = ext4_write_inline_data_end(inode, pos, len,
1409						 copied, page);
1410		if (ret < 0) {
1411			unlock_page(page);
1412			put_page(page);
1413			goto errout;
1414		}
1415		copied = ret;
1416	} else if (unlikely(copied < len) && !PageUptodate(page)) {
1417		copied = 0;
1418		ext4_journalled_zero_new_buffers(handle, inode, page, from, to);
1419	} else {
1420		if (unlikely(copied < len))
1421			ext4_journalled_zero_new_buffers(handle, inode, page,
1422							 from + copied, to);
1423		ret = ext4_walk_page_buffers(handle, inode, page_buffers(page),
1424					     from, from + copied, &partial,
1425					     write_end_fn);
1426		if (!partial)
1427			SetPageUptodate(page);
1428	}
1429	if (!verity)
1430		size_changed = ext4_update_inode_size(inode, pos + copied);
1431	ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1432	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1433	unlock_page(page);
1434	put_page(page);
1435
1436	if (old_size < pos && !verity)
1437		pagecache_isize_extended(inode, old_size, pos);
1438
1439	if (size_changed || inline_data) {
1440		ret2 = ext4_mark_inode_dirty(handle, inode);
1441		if (!ret)
1442			ret = ret2;
1443	}
1444
1445	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1446		/* if we have allocated more blocks and copied
1447		 * less. We will have blocks allocated outside
1448		 * inode->i_size. So truncate them
1449		 */
1450		ext4_orphan_add(handle, inode);
1451
1452errout:
1453	ret2 = ext4_journal_stop(handle);
1454	if (!ret)
1455		ret = ret2;
1456	if (pos + len > inode->i_size && !verity) {
1457		ext4_truncate_failed_write(inode);
1458		/*
1459		 * If truncate failed early the inode might still be
1460		 * on the orphan list; we need to make sure the inode
1461		 * is removed from the orphan list in that case.
1462		 */
1463		if (inode->i_nlink)
1464			ext4_orphan_del(NULL, inode);
1465	}
1466
1467	return ret ? ret : copied;
1468}
1469
1470/*
1471 * Reserve space for a single cluster
1472 */
1473static int ext4_da_reserve_space(struct inode *inode)
1474{
1475	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1476	struct ext4_inode_info *ei = EXT4_I(inode);
1477	int ret;
1478
1479	/*
1480	 * We will charge metadata quota at writeout time; this saves
1481	 * us from metadata over-estimation, though we may go over by
1482	 * a small amount in the end.  Here we just reserve for data.
1483	 */
1484	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1485	if (ret)
1486		return ret;
1487
1488	spin_lock(&ei->i_block_reservation_lock);
1489	if (ext4_claim_free_clusters(sbi, 1, 0)) {
1490		spin_unlock(&ei->i_block_reservation_lock);
1491		dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1492		return -ENOSPC;
1493	}
1494	ei->i_reserved_data_blocks++;
1495	trace_ext4_da_reserve_space(inode);
1496	spin_unlock(&ei->i_block_reservation_lock);
1497
1498	return 0;       /* success */
1499}
1500
1501void ext4_da_release_space(struct inode *inode, int to_free)
1502{
1503	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1504	struct ext4_inode_info *ei = EXT4_I(inode);
1505
1506	if (!to_free)
1507		return;		/* Nothing to release, exit */
1508
1509	spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1510
1511	trace_ext4_da_release_space(inode, to_free);
1512	if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1513		/*
1514		 * if there aren't enough reserved blocks, then the
1515		 * counter is messed up somewhere.  Since this
1516		 * function is called from invalidate page, it's
1517		 * harmless to return without any action.
1518		 */
1519		ext4_warning(inode->i_sb, "ext4_da_release_space: "
1520			 "ino %lu, to_free %d with only %d reserved "
1521			 "data blocks", inode->i_ino, to_free,
1522			 ei->i_reserved_data_blocks);
1523		WARN_ON(1);
1524		to_free = ei->i_reserved_data_blocks;
1525	}
1526	ei->i_reserved_data_blocks -= to_free;
1527
1528	/* update fs dirty data blocks counter */
1529	percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1530
1531	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1532
1533	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1534}
1535
1536/*
1537 * Delayed allocation stuff
1538 */
1539
1540struct mpage_da_data {
1541	struct inode *inode;
1542	struct writeback_control *wbc;
1543
1544	pgoff_t first_page;	/* The first page to write */
1545	pgoff_t next_page;	/* Current page to examine */
1546	pgoff_t last_page;	/* Last page to examine */
1547	/*
1548	 * Extent to map - this can be after first_page because that can be
1549	 * fully mapped. We somewhat abuse m_flags to store whether the extent
1550	 * is delalloc or unwritten.
1551	 */
1552	struct ext4_map_blocks map;
1553	struct ext4_io_submit io_submit;	/* IO submission data */
1554	unsigned int do_map:1;
1555	unsigned int scanned_until_end:1;
1556};
1557
1558static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1559				       bool invalidate)
1560{
1561	int nr_pages, i;
1562	pgoff_t index, end;
1563	struct pagevec pvec;
1564	struct inode *inode = mpd->inode;
1565	struct address_space *mapping = inode->i_mapping;
1566
1567	/* This is necessary when next_page == 0. */
1568	if (mpd->first_page >= mpd->next_page)
1569		return;
1570
1571	mpd->scanned_until_end = 0;
1572	index = mpd->first_page;
1573	end   = mpd->next_page - 1;
1574	if (invalidate) {
1575		ext4_lblk_t start, last;
1576		start = index << (PAGE_SHIFT - inode->i_blkbits);
1577		last = end << (PAGE_SHIFT - inode->i_blkbits);
1578		ext4_es_remove_extent(inode, start, last - start + 1);
1579	}
1580
1581	pagevec_init(&pvec);
1582	while (index <= end) {
1583		nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
1584		if (nr_pages == 0)
1585			break;
1586		for (i = 0; i < nr_pages; i++) {
1587			struct page *page = pvec.pages[i];
1588
1589			BUG_ON(!PageLocked(page));
1590			BUG_ON(PageWriteback(page));
1591			if (invalidate) {
1592				if (page_mapped(page))
1593					clear_page_dirty_for_io(page);
1594				block_invalidatepage(page, 0, PAGE_SIZE);
1595				ClearPageUptodate(page);
1596			}
1597			unlock_page(page);
1598		}
1599		pagevec_release(&pvec);
1600	}
1601}
1602
1603static void ext4_print_free_blocks(struct inode *inode)
1604{
1605	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1606	struct super_block *sb = inode->i_sb;
1607	struct ext4_inode_info *ei = EXT4_I(inode);
1608
1609	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1610	       EXT4_C2B(EXT4_SB(inode->i_sb),
1611			ext4_count_free_clusters(sb)));
1612	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1613	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1614	       (long long) EXT4_C2B(EXT4_SB(sb),
1615		percpu_counter_sum(&sbi->s_freeclusters_counter)));
1616	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1617	       (long long) EXT4_C2B(EXT4_SB(sb),
1618		percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1619	ext4_msg(sb, KERN_CRIT, "Block reservation details");
1620	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1621		 ei->i_reserved_data_blocks);
1622	return;
1623}
1624
1625static int ext4_bh_delay_or_unwritten(handle_t *handle, struct inode *inode,
1626				      struct buffer_head *bh)
1627{
1628	return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1629}
1630
1631/*
1632 * ext4_insert_delayed_block - adds a delayed block to the extents status
1633 *                             tree, incrementing the reserved cluster/block
1634 *                             count or making a pending reservation
1635 *                             where needed
1636 *
1637 * @inode - file containing the newly added block
1638 * @lblk - logical block to be added
1639 *
1640 * Returns 0 on success, negative error code on failure.
1641 */
1642static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1643{
1644	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1645	int ret;
1646	bool allocated = false;
1647
1648	/*
1649	 * If the cluster containing lblk is shared with a delayed,
1650	 * written, or unwritten extent in a bigalloc file system, it's
1651	 * already been accounted for and does not need to be reserved.
1652	 * A pending reservation must be made for the cluster if it's
1653	 * shared with a written or unwritten extent and doesn't already
1654	 * have one.  Written and unwritten extents can be purged from the
1655	 * extents status tree if the system is under memory pressure, so
1656	 * it's necessary to examine the extent tree if a search of the
1657	 * extents status tree doesn't get a match.
1658	 */
1659	if (sbi->s_cluster_ratio == 1) {
1660		ret = ext4_da_reserve_space(inode);
1661		if (ret != 0)   /* ENOSPC */
1662			goto errout;
1663	} else {   /* bigalloc */
1664		if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) {
1665			if (!ext4_es_scan_clu(inode,
1666					      &ext4_es_is_mapped, lblk)) {
1667				ret = ext4_clu_mapped(inode,
1668						      EXT4_B2C(sbi, lblk));
1669				if (ret < 0)
1670					goto errout;
1671				if (ret == 0) {
1672					ret = ext4_da_reserve_space(inode);
1673					if (ret != 0)   /* ENOSPC */
1674						goto errout;
1675				} else {
1676					allocated = true;
1677				}
1678			} else {
1679				allocated = true;
1680			}
1681		}
1682	}
1683
1684	ret = ext4_es_insert_delayed_block(inode, lblk, allocated);
1685
1686errout:
1687	return ret;
1688}
1689
1690/*
1691 * This function is grabs code from the very beginning of
1692 * ext4_map_blocks, but assumes that the caller is from delayed write
1693 * time. This function looks up the requested blocks and sets the
1694 * buffer delay bit under the protection of i_data_sem.
1695 */
1696static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1697			      struct ext4_map_blocks *map,
1698			      struct buffer_head *bh)
1699{
1700	struct extent_status es;
1701	int retval;
1702	sector_t invalid_block = ~((sector_t) 0xffff);
1703#ifdef ES_AGGRESSIVE_TEST
1704	struct ext4_map_blocks orig_map;
1705
1706	memcpy(&orig_map, map, sizeof(*map));
1707#endif
1708
1709	if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1710		invalid_block = ~0;
1711
1712	map->m_flags = 0;
1713	ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
1714		  (unsigned long) map->m_lblk);
1715
1716	/* Lookup extent status tree firstly */
1717	if (ext4_es_lookup_extent(inode, iblock, NULL, &es)) {
1718		if (ext4_es_is_hole(&es)) {
1719			retval = 0;
1720			down_read(&EXT4_I(inode)->i_data_sem);
1721			goto add_delayed;
1722		}
1723
1724		/*
1725		 * Delayed extent could be allocated by fallocate.
1726		 * So we need to check it.
1727		 */
1728		if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1729			map_bh(bh, inode->i_sb, invalid_block);
1730			set_buffer_new(bh);
1731			set_buffer_delay(bh);
1732			return 0;
1733		}
1734
1735		map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1736		retval = es.es_len - (iblock - es.es_lblk);
1737		if (retval > map->m_len)
1738			retval = map->m_len;
1739		map->m_len = retval;
1740		if (ext4_es_is_written(&es))
1741			map->m_flags |= EXT4_MAP_MAPPED;
1742		else if (ext4_es_is_unwritten(&es))
1743			map->m_flags |= EXT4_MAP_UNWRITTEN;
1744		else
1745			BUG();
1746
1747#ifdef ES_AGGRESSIVE_TEST
1748		ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1749#endif
1750		return retval;
1751	}
1752
1753	/*
1754	 * Try to see if we can get the block without requesting a new
1755	 * file system block.
1756	 */
1757	down_read(&EXT4_I(inode)->i_data_sem);
1758	if (ext4_has_inline_data(inode))
1759		retval = 0;
1760	else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
1761		retval = ext4_ext_map_blocks(NULL, inode, map, 0);
1762	else
1763		retval = ext4_ind_map_blocks(NULL, inode, map, 0);
1764
1765add_delayed:
1766	if (retval == 0) {
1767		int ret;
1768
1769		/*
1770		 * XXX: __block_prepare_write() unmaps passed block,
1771		 * is it OK?
1772		 */
1773
1774		ret = ext4_insert_delayed_block(inode, map->m_lblk);
1775		if (ret != 0) {
1776			retval = ret;
1777			goto out_unlock;
1778		}
1779
1780		map_bh(bh, inode->i_sb, invalid_block);
1781		set_buffer_new(bh);
1782		set_buffer_delay(bh);
1783	} else if (retval > 0) {
1784		int ret;
1785		unsigned int status;
1786
1787		if (unlikely(retval != map->m_len)) {
1788			ext4_warning(inode->i_sb,
1789				     "ES len assertion failed for inode "
1790				     "%lu: retval %d != map->m_len %d",
1791				     inode->i_ino, retval, map->m_len);
1792			WARN_ON(1);
1793		}
1794
1795		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1796				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1797		ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1798					    map->m_pblk, status);
1799		if (ret != 0)
1800			retval = ret;
1801	}
1802
1803out_unlock:
1804	up_read((&EXT4_I(inode)->i_data_sem));
1805
1806	return retval;
1807}
1808
1809/*
1810 * This is a special get_block_t callback which is used by
1811 * ext4_da_write_begin().  It will either return mapped block or
1812 * reserve space for a single block.
1813 *
1814 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1815 * We also have b_blocknr = -1 and b_bdev initialized properly
1816 *
1817 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1818 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1819 * initialized properly.
1820 */
1821int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1822			   struct buffer_head *bh, int create)
1823{
1824	struct ext4_map_blocks map;
1825	int ret = 0;
1826
1827	BUG_ON(create == 0);
1828	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1829
1830	map.m_lblk = iblock;
1831	map.m_len = 1;
1832
1833	/*
1834	 * first, we need to know whether the block is allocated already
1835	 * preallocated blocks are unmapped but should treated
1836	 * the same as allocated blocks.
1837	 */
1838	ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1839	if (ret <= 0)
1840		return ret;
1841
1842	map_bh(bh, inode->i_sb, map.m_pblk);
1843	ext4_update_bh_state(bh, map.m_flags);
1844
1845	if (buffer_unwritten(bh)) {
1846		/* A delayed write to unwritten bh should be marked
1847		 * new and mapped.  Mapped ensures that we don't do
1848		 * get_block multiple times when we write to the same
1849		 * offset and new ensures that we do proper zero out
1850		 * for partial write.
1851		 */
1852		set_buffer_new(bh);
1853		set_buffer_mapped(bh);
1854	}
1855	return 0;
1856}
1857
1858static int bget_one(handle_t *handle, struct inode *inode,
1859		    struct buffer_head *bh)
1860{
1861	get_bh(bh);
1862	return 0;
1863}
1864
1865static int bput_one(handle_t *handle, struct inode *inode,
1866		    struct buffer_head *bh)
1867{
1868	put_bh(bh);
1869	return 0;
1870}
1871
1872static int __ext4_journalled_writepage(struct page *page,
1873				       unsigned int len)
1874{
1875	struct address_space *mapping = page->mapping;
1876	struct inode *inode = mapping->host;
1877	struct buffer_head *page_bufs = NULL;
1878	handle_t *handle = NULL;
1879	int ret = 0, err = 0;
1880	int inline_data = ext4_has_inline_data(inode);
1881	struct buffer_head *inode_bh = NULL;
1882
1883	ClearPageChecked(page);
1884
1885	if (inline_data) {
1886		BUG_ON(page->index != 0);
1887		BUG_ON(len > ext4_get_max_inline_size(inode));
1888		inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1889		if (inode_bh == NULL)
1890			goto out;
1891	} else {
1892		page_bufs = page_buffers(page);
1893		if (!page_bufs) {
1894			BUG();
1895			goto out;
1896		}
1897		ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1898				       NULL, bget_one);
1899	}
1900	/*
1901	 * We need to release the page lock before we start the
1902	 * journal, so grab a reference so the page won't disappear
1903	 * out from under us.
1904	 */
1905	get_page(page);
1906	unlock_page(page);
1907
1908	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
1909				    ext4_writepage_trans_blocks(inode));
1910	if (IS_ERR(handle)) {
1911		ret = PTR_ERR(handle);
1912		put_page(page);
1913		goto out_no_pagelock;
1914	}
1915	BUG_ON(!ext4_handle_valid(handle));
1916
1917	lock_page(page);
1918	put_page(page);
1919	if (page->mapping != mapping) {
1920		/* The page got truncated from under us */
1921		ext4_journal_stop(handle);
1922		ret = 0;
1923		goto out;
1924	}
1925
1926	if (inline_data) {
1927		ret = ext4_mark_inode_dirty(handle, inode);
1928	} else {
1929		ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1930					     NULL, do_journal_get_write_access);
1931
1932		err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1933					     NULL, write_end_fn);
1934	}
1935	if (ret == 0)
1936		ret = err;
1937	err = ext4_jbd2_inode_add_write(handle, inode, page_offset(page), len);
1938	if (ret == 0)
1939		ret = err;
1940	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1941	err = ext4_journal_stop(handle);
1942	if (!ret)
1943		ret = err;
1944
1945	ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1946out:
1947	unlock_page(page);
1948out_no_pagelock:
1949	if (!inline_data && page_bufs)
1950		ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len,
1951				       NULL, bput_one);
1952	brelse(inode_bh);
1953	return ret;
1954}
1955
1956/*
1957 * Note that we don't need to start a transaction unless we're journaling data
1958 * because we should have holes filled from ext4_page_mkwrite(). We even don't
1959 * need to file the inode to the transaction's list in ordered mode because if
1960 * we are writing back data added by write(), the inode is already there and if
1961 * we are writing back data modified via mmap(), no one guarantees in which
1962 * transaction the data will hit the disk. In case we are journaling data, we
1963 * cannot start transaction directly because transaction start ranks above page
1964 * lock so we have to do some magic.
1965 *
1966 * This function can get called via...
1967 *   - ext4_writepages after taking page lock (have journal handle)
1968 *   - journal_submit_inode_data_buffers (no journal handle)
1969 *   - shrink_page_list via the kswapd/direct reclaim (no journal handle)
1970 *   - grab_page_cache when doing write_begin (have journal handle)
1971 *
1972 * We don't do any block allocation in this function. If we have page with
1973 * multiple blocks we need to write those buffer_heads that are mapped. This
1974 * is important for mmaped based write. So if we do with blocksize 1K
1975 * truncate(f, 1024);
1976 * a = mmap(f, 0, 4096);
1977 * a[0] = 'a';
1978 * truncate(f, 4096);
1979 * we have in the page first buffer_head mapped via page_mkwrite call back
1980 * but other buffer_heads would be unmapped but dirty (dirty done via the
1981 * do_wp_page). So writepage should write the first block. If we modify
1982 * the mmap area beyond 1024 we will again get a page_fault and the
1983 * page_mkwrite callback will do the block allocation and mark the
1984 * buffer_heads mapped.
1985 *
1986 * We redirty the page if we have any buffer_heads that is either delay or
1987 * unwritten in the page.
1988 *
1989 * We can get recursively called as show below.
1990 *
1991 *	ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
1992 *		ext4_writepage()
1993 *
1994 * But since we don't do any block allocation we should not deadlock.
1995 * Page also have the dirty flag cleared so we don't get recurive page_lock.
1996 */
1997static int ext4_writepage(struct page *page,
1998			  struct writeback_control *wbc)
1999{
2000	int ret = 0;
2001	loff_t size;
2002	unsigned int len;
2003	struct buffer_head *page_bufs = NULL;
2004	struct inode *inode = page->mapping->host;
2005	struct ext4_io_submit io_submit;
2006	bool keep_towrite = false;
2007
2008	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
2009		inode->i_mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
2010		unlock_page(page);
2011		return -EIO;
2012	}
2013
2014	trace_ext4_writepage(page);
2015	size = i_size_read(inode);
2016	if (page->index == size >> PAGE_SHIFT &&
2017	    !ext4_verity_in_progress(inode))
2018		len = size & ~PAGE_MASK;
2019	else
2020		len = PAGE_SIZE;
2021
2022	page_bufs = page_buffers(page);
2023	/*
2024	 * We cannot do block allocation or other extent handling in this
2025	 * function. If there are buffers needing that, we have to redirty
2026	 * the page. But we may reach here when we do a journal commit via
2027	 * journal_submit_inode_data_buffers() and in that case we must write
2028	 * allocated buffers to achieve data=ordered mode guarantees.
2029	 *
2030	 * Also, if there is only one buffer per page (the fs block
2031	 * size == the page size), if one buffer needs block
2032	 * allocation or needs to modify the extent tree to clear the
2033	 * unwritten flag, we know that the page can't be written at
2034	 * all, so we might as well refuse the write immediately.
2035	 * Unfortunately if the block size != page size, we can't as
2036	 * easily detect this case using ext4_walk_page_buffers(), but
2037	 * for the extremely common case, this is an optimization that
2038	 * skips a useless round trip through ext4_bio_write_page().
2039	 */
2040	if (ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len, NULL,
2041				   ext4_bh_delay_or_unwritten)) {
2042		redirty_page_for_writepage(wbc, page);
2043		if ((current->flags & PF_MEMALLOC) ||
2044		    (inode->i_sb->s_blocksize == PAGE_SIZE)) {
2045			/*
2046			 * For memory cleaning there's no point in writing only
2047			 * some buffers. So just bail out. Warn if we came here
2048			 * from direct reclaim.
2049			 */
2050			WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2051							== PF_MEMALLOC);
2052			unlock_page(page);
2053			return 0;
2054		}
2055		keep_towrite = true;
2056	}
2057
2058	if (PageChecked(page) && ext4_should_journal_data(inode))
2059		/*
2060		 * It's mmapped pagecache.  Add buffers and journal it.  There
2061		 * doesn't seem much point in redirtying the page here.
2062		 */
2063		return __ext4_journalled_writepage(page, len);
2064
2065	ext4_io_submit_init(&io_submit, wbc);
2066	io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2067	if (!io_submit.io_end) {
2068		redirty_page_for_writepage(wbc, page);
2069		unlock_page(page);
2070		return -ENOMEM;
2071	}
2072	ret = ext4_bio_write_page(&io_submit, page, len, keep_towrite);
2073	ext4_io_submit(&io_submit);
2074	/* Drop io_end reference we got from init */
2075	ext4_put_io_end_defer(io_submit.io_end);
2076	return ret;
2077}
2078
2079static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2080{
2081	int len;
2082	loff_t size;
2083	int err;
2084
2085	BUG_ON(page->index != mpd->first_page);
2086	clear_page_dirty_for_io(page);
2087	/*
2088	 * We have to be very careful here!  Nothing protects writeback path
2089	 * against i_size changes and the page can be writeably mapped into
2090	 * page tables. So an application can be growing i_size and writing
2091	 * data through mmap while writeback runs. clear_page_dirty_for_io()
2092	 * write-protects our page in page tables and the page cannot get
2093	 * written to again until we release page lock. So only after
2094	 * clear_page_dirty_for_io() we are safe to sample i_size for
2095	 * ext4_bio_write_page() to zero-out tail of the written page. We rely
2096	 * on the barrier provided by TestClearPageDirty in
2097	 * clear_page_dirty_for_io() to make sure i_size is really sampled only
2098	 * after page tables are updated.
2099	 */
2100	size = i_size_read(mpd->inode);
2101	if (page->index == size >> PAGE_SHIFT &&
2102	    !ext4_verity_in_progress(mpd->inode))
2103		len = size & ~PAGE_MASK;
2104	else
2105		len = PAGE_SIZE;
2106	err = ext4_bio_write_page(&mpd->io_submit, page, len, false);
2107	if (!err)
2108		mpd->wbc->nr_to_write--;
2109	mpd->first_page++;
2110
2111	return err;
2112}
2113
2114#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
2115
2116/*
2117 * mballoc gives us at most this number of blocks...
2118 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2119 * The rest of mballoc seems to handle chunks up to full group size.
2120 */
2121#define MAX_WRITEPAGES_EXTENT_LEN 2048
2122
2123/*
2124 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2125 *
2126 * @mpd - extent of blocks
2127 * @lblk - logical number of the block in the file
2128 * @bh - buffer head we want to add to the extent
2129 *
2130 * The function is used to collect contig. blocks in the same state. If the
2131 * buffer doesn't require mapping for writeback and we haven't started the
2132 * extent of buffers to map yet, the function returns 'true' immediately - the
2133 * caller can write the buffer right away. Otherwise the function returns true
2134 * if the block has been added to the extent, false if the block couldn't be
2135 * added.
2136 */
2137static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2138				   struct buffer_head *bh)
2139{
2140	struct ext4_map_blocks *map = &mpd->map;
2141
2142	/* Buffer that doesn't need mapping for writeback? */
2143	if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2144	    (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2145		/* So far no extent to map => we write the buffer right away */
2146		if (map->m_len == 0)
2147			return true;
2148		return false;
2149	}
2150
2151	/* First block in the extent? */
2152	if (map->m_len == 0) {
2153		/* We cannot map unless handle is started... */
2154		if (!mpd->do_map)
2155			return false;
2156		map->m_lblk = lblk;
2157		map->m_len = 1;
2158		map->m_flags = bh->b_state & BH_FLAGS;
2159		return true;
2160	}
2161
2162	/* Don't go larger than mballoc is willing to allocate */
2163	if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2164		return false;
2165
2166	/* Can we merge the block to our big extent? */
2167	if (lblk == map->m_lblk + map->m_len &&
2168	    (bh->b_state & BH_FLAGS) == map->m_flags) {
2169		map->m_len++;
2170		return true;
2171	}
2172	return false;
2173}
2174
2175/*
2176 * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2177 *
2178 * @mpd - extent of blocks for mapping
2179 * @head - the first buffer in the page
2180 * @bh - buffer we should start processing from
2181 * @lblk - logical number of the block in the file corresponding to @bh
2182 *
2183 * Walk through page buffers from @bh upto @head (exclusive) and either submit
2184 * the page for IO if all buffers in this page were mapped and there's no
2185 * accumulated extent of buffers to map or add buffers in the page to the
2186 * extent of buffers to map. The function returns 1 if the caller can continue
2187 * by processing the next page, 0 if it should stop adding buffers to the
2188 * extent to map because we cannot extend it anymore. It can also return value
2189 * < 0 in case of error during IO submission.
2190 */
2191static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2192				   struct buffer_head *head,
2193				   struct buffer_head *bh,
2194				   ext4_lblk_t lblk)
2195{
2196	struct inode *inode = mpd->inode;
2197	int err;
2198	ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2199							>> inode->i_blkbits;
2200
2201	if (ext4_verity_in_progress(inode))
2202		blocks = EXT_MAX_BLOCKS;
2203
2204	do {
2205		BUG_ON(buffer_locked(bh));
2206
2207		if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2208			/* Found extent to map? */
2209			if (mpd->map.m_len)
2210				return 0;
2211			/* Buffer needs mapping and handle is not started? */
2212			if (!mpd->do_map)
2213				return 0;
2214			/* Everything mapped so far and we hit EOF */
2215			break;
2216		}
2217	} while (lblk++, (bh = bh->b_this_page) != head);
2218	/* So far everything mapped? Submit the page for IO. */
2219	if (mpd->map.m_len == 0) {
2220		err = mpage_submit_page(mpd, head->b_page);
2221		if (err < 0)
2222			return err;
2223	}
2224	if (lblk >= blocks) {
2225		mpd->scanned_until_end = 1;
2226		return 0;
2227	}
2228	return 1;
2229}
2230
2231/*
2232 * mpage_process_page - update page buffers corresponding to changed extent and
2233 *		       may submit fully mapped page for IO
2234 *
2235 * @mpd		- description of extent to map, on return next extent to map
2236 * @m_lblk	- logical block mapping.
2237 * @m_pblk	- corresponding physical mapping.
2238 * @map_bh	- determines on return whether this page requires any further
2239 *		  mapping or not.
2240 * Scan given page buffers corresponding to changed extent and update buffer
2241 * state according to new extent state.
2242 * We map delalloc buffers to their physical location, clear unwritten bits.
2243 * If the given page is not fully mapped, we update @map to the next extent in
2244 * the given page that needs mapping & return @map_bh as true.
2245 */
2246static int mpage_process_page(struct mpage_da_data *mpd, struct page *page,
2247			      ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2248			      bool *map_bh)
2249{
2250	struct buffer_head *head, *bh;
2251	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2252	ext4_lblk_t lblk = *m_lblk;
2253	ext4_fsblk_t pblock = *m_pblk;
2254	int err = 0;
2255	int blkbits = mpd->inode->i_blkbits;
2256	ssize_t io_end_size = 0;
2257	struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2258
2259	bh = head = page_buffers(page);
2260	do {
2261		if (lblk < mpd->map.m_lblk)
2262			continue;
2263		if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2264			/*
2265			 * Buffer after end of mapped extent.
2266			 * Find next buffer in the page to map.
2267			 */
2268			mpd->map.m_len = 0;
2269			mpd->map.m_flags = 0;
2270			io_end_vec->size += io_end_size;
2271			io_end_size = 0;
2272
2273			err = mpage_process_page_bufs(mpd, head, bh, lblk);
2274			if (err > 0)
2275				err = 0;
2276			if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2277				io_end_vec = ext4_alloc_io_end_vec(io_end);
2278				if (IS_ERR(io_end_vec)) {
2279					err = PTR_ERR(io_end_vec);
2280					goto out;
2281				}
2282				io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2283			}
2284			*map_bh = true;
2285			goto out;
2286		}
2287		if (buffer_delay(bh)) {
2288			clear_buffer_delay(bh);
2289			bh->b_blocknr = pblock++;
2290		}
2291		clear_buffer_unwritten(bh);
2292		io_end_size += (1 << blkbits);
2293	} while (lblk++, (bh = bh->b_this_page) != head);
2294
2295	io_end_vec->size += io_end_size;
2296	io_end_size = 0;
2297	*map_bh = false;
2298out:
2299	*m_lblk = lblk;
2300	*m_pblk = pblock;
2301	return err;
2302}
2303
2304/*
2305 * mpage_map_buffers - update buffers corresponding to changed extent and
2306 *		       submit fully mapped pages for IO
2307 *
2308 * @mpd - description of extent to map, on return next extent to map
2309 *
2310 * Scan buffers corresponding to changed extent (we expect corresponding pages
2311 * to be already locked) and update buffer state according to new extent state.
2312 * We map delalloc buffers to their physical location, clear unwritten bits,
2313 * and mark buffers as uninit when we perform writes to unwritten extents
2314 * and do extent conversion after IO is finished. If the last page is not fully
2315 * mapped, we update @map to the next extent in the last page that needs
2316 * mapping. Otherwise we submit the page for IO.
2317 */
2318static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2319{
2320	struct pagevec pvec;
2321	int nr_pages, i;
2322	struct inode *inode = mpd->inode;
2323	int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2324	pgoff_t start, end;
2325	ext4_lblk_t lblk;
2326	ext4_fsblk_t pblock;
2327	int err;
2328	bool map_bh = false;
2329
2330	start = mpd->map.m_lblk >> bpp_bits;
2331	end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2332	lblk = start << bpp_bits;
2333	pblock = mpd->map.m_pblk;
2334
2335	pagevec_init(&pvec);
2336	while (start <= end) {
2337		nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
2338						&start, end);
2339		if (nr_pages == 0)
2340			break;
2341		for (i = 0; i < nr_pages; i++) {
2342			struct page *page = pvec.pages[i];
2343
2344			err = mpage_process_page(mpd, page, &lblk, &pblock,
2345						 &map_bh);
2346			/*
2347			 * If map_bh is true, means page may require further bh
2348			 * mapping, or maybe the page was submitted for IO.
2349			 * So we return to call further extent mapping.
2350			 */
2351			if (err < 0 || map_bh)
2352				goto out;
2353			/* Page fully mapped - let IO run! */
2354			err = mpage_submit_page(mpd, page);
2355			if (err < 0)
2356				goto out;
2357		}
2358		pagevec_release(&pvec);
2359	}
2360	/* Extent fully mapped and matches with page boundary. We are done. */
2361	mpd->map.m_len = 0;
2362	mpd->map.m_flags = 0;
2363	return 0;
2364out:
2365	pagevec_release(&pvec);
2366	return err;
2367}
2368
2369static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2370{
2371	struct inode *inode = mpd->inode;
2372	struct ext4_map_blocks *map = &mpd->map;
2373	int get_blocks_flags;
2374	int err, dioread_nolock;
2375
2376	trace_ext4_da_write_pages_extent(inode, map);
2377	/*
2378	 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2379	 * to convert an unwritten extent to be initialized (in the case
2380	 * where we have written into one or more preallocated blocks).  It is
2381	 * possible that we're going to need more metadata blocks than
2382	 * previously reserved. However we must not fail because we're in
2383	 * writeback and there is nothing we can do about it so it might result
2384	 * in data loss.  So use reserved blocks to allocate metadata if
2385	 * possible.
2386	 *
2387	 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2388	 * the blocks in question are delalloc blocks.  This indicates
2389	 * that the blocks and quotas has already been checked when
2390	 * the data was copied into the page cache.
2391	 */
2392	get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2393			   EXT4_GET_BLOCKS_METADATA_NOFAIL |
2394			   EXT4_GET_BLOCKS_IO_SUBMIT;
2395	dioread_nolock = ext4_should_dioread_nolock(inode);
2396	if (dioread_nolock)
2397		get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2398	if (map->m_flags & BIT(BH_Delay))
2399		get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2400
2401	err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2402	if (err < 0)
2403		return err;
2404	if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2405		if (!mpd->io_submit.io_end->handle &&
2406		    ext4_handle_valid(handle)) {
2407			mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2408			handle->h_rsv_handle = NULL;
2409		}
2410		ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2411	}
2412
2413	BUG_ON(map->m_len == 0);
2414	return 0;
2415}
2416
2417/*
2418 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2419 *				 mpd->len and submit pages underlying it for IO
2420 *
2421 * @handle - handle for journal operations
2422 * @mpd - extent to map
2423 * @give_up_on_write - we set this to true iff there is a fatal error and there
2424 *                     is no hope of writing the data. The caller should discard
2425 *                     dirty pages to avoid infinite loops.
2426 *
2427 * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2428 * delayed, blocks are allocated, if it is unwritten, we may need to convert
2429 * them to initialized or split the described range from larger unwritten
2430 * extent. Note that we need not map all the described range since allocation
2431 * can return less blocks or the range is covered by more unwritten extents. We
2432 * cannot map more because we are limited by reserved transaction credits. On
2433 * the other hand we always make sure that the last touched page is fully
2434 * mapped so that it can be written out (and thus forward progress is
2435 * guaranteed). After mapping we submit all mapped pages for IO.
2436 */
2437static int mpage_map_and_submit_extent(handle_t *handle,
2438				       struct mpage_da_data *mpd,
2439				       bool *give_up_on_write)
2440{
2441	struct inode *inode = mpd->inode;
2442	struct ext4_map_blocks *map = &mpd->map;
2443	int err;
2444	loff_t disksize;
2445	int progress = 0;
2446	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2447	struct ext4_io_end_vec *io_end_vec;
2448
2449	io_end_vec = ext4_alloc_io_end_vec(io_end);
2450	if (IS_ERR(io_end_vec))
2451		return PTR_ERR(io_end_vec);
2452	io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
2453	do {
2454		err = mpage_map_one_extent(handle, mpd);
2455		if (err < 0) {
2456			struct super_block *sb = inode->i_sb;
2457
2458			if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2459			    ext4_test_mount_flag(sb, EXT4_MF_FS_ABORTED))
2460				goto invalidate_dirty_pages;
2461			/*
2462			 * Let the uper layers retry transient errors.
2463			 * In the case of ENOSPC, if ext4_count_free_blocks()
2464			 * is non-zero, a commit should free up blocks.
2465			 */
2466			if ((err == -ENOMEM) ||
2467			    (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2468				if (progress)
2469					goto update_disksize;
2470				return err;
2471			}
2472			ext4_msg(sb, KERN_CRIT,
2473				 "Delayed block allocation failed for "
2474				 "inode %lu at logical offset %llu with"
2475				 " max blocks %u with error %d",
2476				 inode->i_ino,
2477				 (unsigned long long)map->m_lblk,
2478				 (unsigned)map->m_len, -err);
2479			ext4_msg(sb, KERN_CRIT,
2480				 "This should not happen!! Data will "
2481				 "be lost\n");
2482			if (err == -ENOSPC)
2483				ext4_print_free_blocks(inode);
2484		invalidate_dirty_pages:
2485			*give_up_on_write = true;
2486			return err;
2487		}
2488		progress = 1;
2489		/*
2490		 * Update buffer state, submit mapped pages, and get us new
2491		 * extent to map
2492		 */
2493		err = mpage_map_and_submit_buffers(mpd);
2494		if (err < 0)
2495			goto update_disksize;
2496	} while (map->m_len);
2497
2498update_disksize:
2499	/*
2500	 * Update on-disk size after IO is submitted.  Races with
2501	 * truncate are avoided by checking i_size under i_data_sem.
2502	 */
2503	disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2504	if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2505		int err2;
2506		loff_t i_size;
2507
2508		down_write(&EXT4_I(inode)->i_data_sem);
2509		i_size = i_size_read(inode);
2510		if (disksize > i_size)
2511			disksize = i_size;
2512		if (disksize > EXT4_I(inode)->i_disksize)
2513			EXT4_I(inode)->i_disksize = disksize;
2514		up_write(&EXT4_I(inode)->i_data_sem);
2515		err2 = ext4_mark_inode_dirty(handle, inode);
2516		if (err2) {
2517			ext4_error_err(inode->i_sb, -err2,
2518				       "Failed to mark inode %lu dirty",
2519				       inode->i_ino);
2520		}
2521		if (!err)
2522			err = err2;
2523	}
2524	return err;
2525}
2526
2527/*
2528 * Calculate the total number of credits to reserve for one writepages
2529 * iteration. This is called from ext4_writepages(). We map an extent of
2530 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
2531 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2532 * bpp - 1 blocks in bpp different extents.
2533 */
2534static int ext4_da_writepages_trans_blocks(struct inode *inode)
2535{
2536	int bpp = ext4_journal_blocks_per_page(inode);
2537
2538	return ext4_meta_trans_blocks(inode,
2539				MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
2540}
2541
2542/*
2543 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2544 * 				 and underlying extent to map
2545 *
2546 * @mpd - where to look for pages
2547 *
2548 * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2549 * IO immediately. When we find a page which isn't mapped we start accumulating
2550 * extent of buffers underlying these pages that needs mapping (formed by
2551 * either delayed or unwritten buffers). We also lock the pages containing
2552 * these buffers. The extent found is returned in @mpd structure (starting at
2553 * mpd->lblk with length mpd->len blocks).
2554 *
2555 * Note that this function can attach bios to one io_end structure which are
2556 * neither logically nor physically contiguous. Although it may seem as an
2557 * unnecessary complication, it is actually inevitable in blocksize < pagesize
2558 * case as we need to track IO to all buffers underlying a page in one io_end.
2559 */
2560static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2561{
2562	struct address_space *mapping = mpd->inode->i_mapping;
2563	struct pagevec pvec;
2564	unsigned int nr_pages;
2565	long left = mpd->wbc->nr_to_write;
2566	pgoff_t index = mpd->first_page;
2567	pgoff_t end = mpd->last_page;
2568	xa_mark_t tag;
2569	int i, err = 0;
2570	int blkbits = mpd->inode->i_blkbits;
2571	ext4_lblk_t lblk;
2572	struct buffer_head *head;
2573
2574	if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2575		tag = PAGECACHE_TAG_TOWRITE;
2576	else
2577		tag = PAGECACHE_TAG_DIRTY;
2578
2579	pagevec_init(&pvec);
2580	mpd->map.m_len = 0;
2581	mpd->next_page = index;
2582	while (index <= end) {
2583		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2584				tag);
2585		if (nr_pages == 0)
2586			break;
2587
2588		for (i = 0; i < nr_pages; i++) {
2589			struct page *page = pvec.pages[i];
2590
2591			/*
2592			 * Accumulated enough dirty pages? This doesn't apply
2593			 * to WB_SYNC_ALL mode. For integrity sync we have to
2594			 * keep going because someone may be concurrently
2595			 * dirtying pages, and we might have synced a lot of
2596			 * newly appeared dirty pages, but have not synced all
2597			 * of the old dirty pages.
2598			 */
2599			if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2600				goto out;
2601
2602			/* If we can't merge this page, we are done. */
2603			if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2604				goto out;
2605
2606			lock_page(page);
2607			/*
2608			 * If the page is no longer dirty, or its mapping no
2609			 * longer corresponds to inode we are writing (which
2610			 * means it has been truncated or invalidated), or the
2611			 * page is already under writeback and we are not doing
2612			 * a data integrity writeback, skip the page
2613			 */
2614			if (!PageDirty(page) ||
2615			    (PageWriteback(page) &&
2616			     (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2617			    unlikely(page->mapping != mapping)) {
2618				unlock_page(page);
2619				continue;
2620			}
2621
2622			wait_on_page_writeback(page);
2623			BUG_ON(PageWriteback(page));
2624
2625			if (mpd->map.m_len == 0)
2626				mpd->first_page = page->index;
2627			mpd->next_page = page->index + 1;
2628			/* Add all dirty buffers to mpd */
2629			lblk = ((ext4_lblk_t)page->index) <<
2630				(PAGE_SHIFT - blkbits);
2631			head = page_buffers(page);
2632			err = mpage_process_page_bufs(mpd, head, head, lblk);
2633			if (err <= 0)
2634				goto out;
2635			err = 0;
2636			left--;
2637		}
2638		pagevec_release(&pvec);
2639		cond_resched();
2640	}
2641	mpd->scanned_until_end = 1;
2642	return 0;
2643out:
2644	pagevec_release(&pvec);
2645	return err;
2646}
2647
2648static int ext4_writepages(struct address_space *mapping,
2649			   struct writeback_control *wbc)
2650{
2651	pgoff_t	writeback_index = 0;
2652	long nr_to_write = wbc->nr_to_write;
2653	int range_whole = 0;
2654	int cycled = 1;
2655	handle_t *handle = NULL;
2656	struct mpage_da_data mpd;
2657	struct inode *inode = mapping->host;
2658	int needed_blocks, rsv_blocks = 0, ret = 0;
2659	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2660	struct blk_plug plug;
2661	bool give_up_on_write = false;
2662
2663	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2664		return -EIO;
2665
2666	percpu_down_read(&sbi->s_writepages_rwsem);
2667	trace_ext4_writepages(inode, wbc);
2668
2669	/*
2670	 * No pages to write? This is mainly a kludge to avoid starting
2671	 * a transaction for special inodes like journal inode on last iput()
2672	 * because that could violate lock ordering on umount
2673	 */
2674	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2675		goto out_writepages;
2676
2677	if (ext4_should_journal_data(inode)) {
2678		ret = generic_writepages(mapping, wbc);
2679		goto out_writepages;
2680	}
2681
2682	/*
2683	 * If the filesystem has aborted, it is read-only, so return
2684	 * right away instead of dumping stack traces later on that
2685	 * will obscure the real source of the problem.  We test
2686	 * EXT4_MF_FS_ABORTED instead of sb->s_flag's SB_RDONLY because
2687	 * the latter could be true if the filesystem is mounted
2688	 * read-only, and in that case, ext4_writepages should
2689	 * *never* be called, so if that ever happens, we would want
2690	 * the stack trace.
2691	 */
2692	if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2693		     ext4_test_mount_flag(inode->i_sb, EXT4_MF_FS_ABORTED))) {
2694		ret = -EROFS;
2695		goto out_writepages;
2696	}
2697
2698	/*
2699	 * If we have inline data and arrive here, it means that
2700	 * we will soon create the block for the 1st page, so
2701	 * we'd better clear the inline data here.
2702	 */
2703	if (ext4_has_inline_data(inode)) {
2704		/* Just inode will be modified... */
2705		handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2706		if (IS_ERR(handle)) {
2707			ret = PTR_ERR(handle);
2708			goto out_writepages;
2709		}
2710		BUG_ON(ext4_test_inode_state(inode,
2711				EXT4_STATE_MAY_INLINE_DATA));
2712		ext4_destroy_inline_data(handle, inode);
2713		ext4_journal_stop(handle);
2714	}
2715
2716	if (ext4_should_dioread_nolock(inode)) {
2717		/*
2718		 * We may need to convert up to one extent per block in
2719		 * the page and we may dirty the inode.
2720		 */
2721		rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2722						PAGE_SIZE >> inode->i_blkbits);
2723	}
2724
2725	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2726		range_whole = 1;
2727
2728	if (wbc->range_cyclic) {
2729		writeback_index = mapping->writeback_index;
2730		if (writeback_index)
2731			cycled = 0;
2732		mpd.first_page = writeback_index;
2733		mpd.last_page = -1;
2734	} else {
2735		mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2736		mpd.last_page = wbc->range_end >> PAGE_SHIFT;
2737	}
2738
2739	mpd.inode = inode;
2740	mpd.wbc = wbc;
2741	ext4_io_submit_init(&mpd.io_submit, wbc);
2742retry:
2743	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2744		tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2745	blk_start_plug(&plug);
2746
2747	/*
2748	 * First writeback pages that don't need mapping - we can avoid
2749	 * starting a transaction unnecessarily and also avoid being blocked
2750	 * in the block layer on device congestion while having transaction
2751	 * started.
2752	 */
2753	mpd.do_map = 0;
2754	mpd.scanned_until_end = 0;
2755	mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2756	if (!mpd.io_submit.io_end) {
2757		ret = -ENOMEM;
2758		goto unplug;
2759	}
2760	ret = mpage_prepare_extent_to_map(&mpd);
2761	/* Unlock pages we didn't use */
2762	mpage_release_unused_pages(&mpd, false);
2763	/* Submit prepared bio */
2764	ext4_io_submit(&mpd.io_submit);
2765	ext4_put_io_end_defer(mpd.io_submit.io_end);
2766	mpd.io_submit.io_end = NULL;
2767	if (ret < 0)
2768		goto unplug;
2769
2770	while (!mpd.scanned_until_end && wbc->nr_to_write > 0) {
2771		/* For each extent of pages we use new io_end */
2772		mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2773		if (!mpd.io_submit.io_end) {
2774			ret = -ENOMEM;
2775			break;
2776		}
2777
2778		/*
2779		 * We have two constraints: We find one extent to map and we
2780		 * must always write out whole page (makes a difference when
2781		 * blocksize < pagesize) so that we don't block on IO when we
2782		 * try to write out the rest of the page. Journalled mode is
2783		 * not supported by delalloc.
2784		 */
2785		BUG_ON(ext4_should_journal_data(inode));
2786		needed_blocks = ext4_da_writepages_trans_blocks(inode);
2787
2788		/* start a new transaction */
2789		handle = ext4_journal_start_with_reserve(inode,
2790				EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2791		if (IS_ERR(handle)) {
2792			ret = PTR_ERR(handle);
2793			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2794			       "%ld pages, ino %lu; err %d", __func__,
2795				wbc->nr_to_write, inode->i_ino, ret);
2796			/* Release allocated io_end */
2797			ext4_put_io_end(mpd.io_submit.io_end);
2798			mpd.io_submit.io_end = NULL;
2799			break;
2800		}
2801		mpd.do_map = 1;
2802
2803		trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2804		ret = mpage_prepare_extent_to_map(&mpd);
2805		if (!ret && mpd.map.m_len)
2806			ret = mpage_map_and_submit_extent(handle, &mpd,
2807					&give_up_on_write);
2808		/*
2809		 * Caution: If the handle is synchronous,
2810		 * ext4_journal_stop() can wait for transaction commit
2811		 * to finish which may depend on writeback of pages to
2812		 * complete or on page lock to be released.  In that
2813		 * case, we have to wait until after we have
2814		 * submitted all the IO, released page locks we hold,
2815		 * and dropped io_end reference (for extent conversion
2816		 * to be able to complete) before stopping the handle.
2817		 */
2818		if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2819			ext4_journal_stop(handle);
2820			handle = NULL;
2821			mpd.do_map = 0;
2822		}
2823		/* Unlock pages we didn't use */
2824		mpage_release_unused_pages(&mpd, give_up_on_write);
2825		/* Submit prepared bio */
2826		ext4_io_submit(&mpd.io_submit);
2827
2828		/*
2829		 * Drop our io_end reference we got from init. We have
2830		 * to be careful and use deferred io_end finishing if
2831		 * we are still holding the transaction as we can
2832		 * release the last reference to io_end which may end
2833		 * up doing unwritten extent conversion.
2834		 */
2835		if (handle) {
2836			ext4_put_io_end_defer(mpd.io_submit.io_end);
2837			ext4_journal_stop(handle);
2838		} else
2839			ext4_put_io_end(mpd.io_submit.io_end);
2840		mpd.io_submit.io_end = NULL;
2841
2842		if (ret == -ENOSPC && sbi->s_journal) {
2843			/*
2844			 * Commit the transaction which would
2845			 * free blocks released in the transaction
2846			 * and try again
2847			 */
2848			jbd2_journal_force_commit_nested(sbi->s_journal);
2849			ret = 0;
2850			continue;
2851		}
2852		/* Fatal error - ENOMEM, EIO... */
2853		if (ret)
2854			break;
2855	}
2856unplug:
2857	blk_finish_plug(&plug);
2858	if (!ret && !cycled && wbc->nr_to_write > 0) {
2859		cycled = 1;
2860		mpd.last_page = writeback_index - 1;
2861		mpd.first_page = 0;
2862		goto retry;
2863	}
2864
2865	/* Update index */
2866	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2867		/*
2868		 * Set the writeback_index so that range_cyclic
2869		 * mode will write it back later
2870		 */
2871		mapping->writeback_index = mpd.first_page;
2872
2873out_writepages:
2874	trace_ext4_writepages_result(inode, wbc, ret,
2875				     nr_to_write - wbc->nr_to_write);
2876	percpu_up_read(&sbi->s_writepages_rwsem);
2877	return ret;
2878}
2879
2880static int ext4_dax_writepages(struct address_space *mapping,
2881			       struct writeback_control *wbc)
2882{
2883	int ret;
2884	long nr_to_write = wbc->nr_to_write;
2885	struct inode *inode = mapping->host;
2886	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2887
2888	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2889		return -EIO;
2890
2891	percpu_down_read(&sbi->s_writepages_rwsem);
2892	trace_ext4_writepages(inode, wbc);
2893
2894	ret = dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc);
2895	trace_ext4_writepages_result(inode, wbc, ret,
2896				     nr_to_write - wbc->nr_to_write);
2897	percpu_up_read(&sbi->s_writepages_rwsem);
2898	return ret;
2899}
2900
2901static int ext4_nonda_switch(struct super_block *sb)
2902{
2903	s64 free_clusters, dirty_clusters;
2904	struct ext4_sb_info *sbi = EXT4_SB(sb);
2905
2906	/*
2907	 * switch to non delalloc mode if we are running low
2908	 * on free block. The free block accounting via percpu
2909	 * counters can get slightly wrong with percpu_counter_batch getting
2910	 * accumulated on each CPU without updating global counters
2911	 * Delalloc need an accurate free block accounting. So switch
2912	 * to non delalloc when we are near to error range.
2913	 */
2914	free_clusters =
2915		percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2916	dirty_clusters =
2917		percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2918	/*
2919	 * Start pushing delalloc when 1/2 of free blocks are dirty.
2920	 */
2921	if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
2922		try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
2923
2924	if (2 * free_clusters < 3 * dirty_clusters ||
2925	    free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
2926		/*
2927		 * free block count is less than 150% of dirty blocks
2928		 * or free blocks is less than watermark
2929		 */
2930		return 1;
2931	}
2932	return 0;
2933}
2934
2935/* We always reserve for an inode update; the superblock could be there too */
2936static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
2937{
2938	if (likely(ext4_has_feature_large_file(inode->i_sb)))
2939		return 1;
2940
2941	if (pos + len <= 0x7fffffffULL)
2942		return 1;
2943
2944	/* We might need to update the superblock to set LARGE_FILE */
2945	return 2;
2946}
2947
2948static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
2949			       loff_t pos, unsigned len, unsigned flags,
2950			       struct page **pagep, void **fsdata)
2951{
2952	int ret, retries = 0;
2953	struct page *page;
2954	pgoff_t index;
2955	struct inode *inode = mapping->host;
2956	handle_t *handle;
2957
2958	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2959		return -EIO;
2960
2961	index = pos >> PAGE_SHIFT;
2962
2963	if (ext4_nonda_switch(inode->i_sb) || S_ISLNK(inode->i_mode) ||
2964	    ext4_verity_in_progress(inode)) {
2965		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2966		return ext4_write_begin(file, mapping, pos,
2967					len, flags, pagep, fsdata);
2968	}
2969	*fsdata = (void *)0;
2970	trace_ext4_da_write_begin(inode, pos, len, flags);
2971
2972	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
2973		ret = ext4_da_write_inline_data_begin(mapping, inode,
2974						      pos, len, flags,
2975						      pagep, fsdata);
2976		if (ret < 0)
2977			return ret;
2978		if (ret == 1)
2979			return 0;
2980	}
2981
2982	/*
2983	 * grab_cache_page_write_begin() can take a long time if the
2984	 * system is thrashing due to memory pressure, or if the page
2985	 * is being written back.  So grab it first before we start
2986	 * the transaction handle.  This also allows us to allocate
2987	 * the page (if needed) without using GFP_NOFS.
2988	 */
2989retry_grab:
2990	page = grab_cache_page_write_begin(mapping, index, flags);
2991	if (!page)
2992		return -ENOMEM;
2993	unlock_page(page);
2994
2995	/*
2996	 * With delayed allocation, we don't log the i_disksize update
2997	 * if there is delayed block allocation. But we still need
2998	 * to journalling the i_disksize update if writes to the end
2999	 * of file which has an already mapped buffer.
3000	 */
3001retry_journal:
3002	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
3003				ext4_da_write_credits(inode, pos, len));
3004	if (IS_ERR(handle)) {
3005		put_page(page);
3006		return PTR_ERR(handle);
3007	}
3008
3009	lock_page(page);
3010	if (page->mapping != mapping) {
3011		/* The page got truncated from under us */
3012		unlock_page(page);
3013		put_page(page);
3014		ext4_journal_stop(handle);
3015		goto retry_grab;
3016	}
3017	/* In case writeback began while the page was unlocked */
3018	wait_for_stable_page(page);
3019
3020#ifdef CONFIG_FS_ENCRYPTION
3021	ret = ext4_block_write_begin(page, pos, len,
3022				     ext4_da_get_block_prep);
3023#else
3024	ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
3025#endif
3026	if (ret < 0) {
3027		unlock_page(page);
3028		ext4_journal_stop(handle);
3029		/*
3030		 * block_write_begin may have instantiated a few blocks
3031		 * outside i_size.  Trim these off again. Don't need
3032		 * i_size_read because we hold i_mutex.
3033		 */
3034		if (pos + len > inode->i_size)
3035			ext4_truncate_failed_write(inode);
3036
3037		if (ret == -ENOSPC &&
3038		    ext4_should_retry_alloc(inode->i_sb, &retries))
3039			goto retry_journal;
3040
3041		put_page(page);
3042		return ret;
3043	}
3044
3045	*pagep = page;
3046	return ret;
3047}
3048
3049/*
3050 * Check if we should update i_disksize
3051 * when write to the end of file but not require block allocation
3052 */
3053static int ext4_da_should_update_i_disksize(struct page *page,
3054					    unsigned long offset)
3055{
3056	struct buffer_head *bh;
3057	struct inode *inode = page->mapping->host;
3058	unsigned int idx;
3059	int i;
3060
3061	bh = page_buffers(page);
3062	idx = offset >> inode->i_blkbits;
3063
3064	for (i = 0; i < idx; i++)
3065		bh = bh->b_this_page;
3066
3067	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3068		return 0;
3069	return 1;
3070}
3071
3072static int ext4_da_write_end(struct file *file,
3073			     struct address_space *mapping,
3074			     loff_t pos, unsigned len, unsigned copied,
3075			     struct page *page, void *fsdata)
3076{
3077	struct inode *inode = mapping->host;
3078	int ret = 0, ret2;
3079	handle_t *handle = ext4_journal_current_handle();
3080	loff_t new_i_size;
3081	unsigned long start, end;
3082	int write_mode = (int)(unsigned long)fsdata;
3083
3084	if (write_mode == FALL_BACK_TO_NONDELALLOC)
3085		return ext4_write_end(file, mapping, pos,
3086				      len, copied, page, fsdata);
3087
3088	trace_ext4_da_write_end(inode, pos, len, copied);
3089	start = pos & (PAGE_SIZE - 1);
3090	end = start + copied - 1;
3091
3092	/*
3093	 * generic_write_end() will run mark_inode_dirty() if i_size
3094	 * changes.  So let's piggyback the i_disksize mark_inode_dirty
3095	 * into that.
3096	 */
3097	new_i_size = pos + copied;
3098	if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
3099		if (ext4_has_inline_data(inode) ||
3100		    ext4_da_should_update_i_disksize(page, end)) {
3101			ext4_update_i_disksize(inode, new_i_size);
3102			/* We need to mark inode dirty even if
3103			 * new_i_size is less that inode->i_size
3104			 * bu greater than i_disksize.(hint delalloc)
3105			 */
3106			ret = ext4_mark_inode_dirty(handle, inode);
3107		}
3108	}
3109
3110	if (write_mode != CONVERT_INLINE_DATA &&
3111	    ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3112	    ext4_has_inline_data(inode))
3113		ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
3114						     page);
3115	else
3116		ret2 = generic_write_end(file, mapping, pos, len, copied,
3117							page, fsdata);
3118
3119	copied = ret2;
3120	if (ret2 < 0)
3121		ret = ret2;
3122	ret2 = ext4_journal_stop(handle);
3123	if (unlikely(ret2 && !ret))
3124		ret = ret2;
3125
3126	return ret ? ret : copied;
3127}
3128
3129/*
3130 * Force all delayed allocation blocks to be allocated for a given inode.
3131 */
3132int ext4_alloc_da_blocks(struct inode *inode)
3133{
3134	trace_ext4_alloc_da_blocks(inode);
3135
3136	if (!EXT4_I(inode)->i_reserved_data_blocks)
3137		return 0;
3138
3139	/*
3140	 * We do something simple for now.  The filemap_flush() will
3141	 * also start triggering a write of the data blocks, which is
3142	 * not strictly speaking necessary (and for users of
3143	 * laptop_mode, not even desirable).  However, to do otherwise
3144	 * would require replicating code paths in:
3145	 *
3146	 * ext4_writepages() ->
3147	 *    write_cache_pages() ---> (via passed in callback function)
3148	 *        __mpage_da_writepage() -->
3149	 *           mpage_add_bh_to_extent()
3150	 *           mpage_da_map_blocks()
3151	 *
3152	 * The problem is that write_cache_pages(), located in
3153	 * mm/page-writeback.c, marks pages clean in preparation for
3154	 * doing I/O, which is not desirable if we're not planning on
3155	 * doing I/O at all.
3156	 *
3157	 * We could call write_cache_pages(), and then redirty all of
3158	 * the pages by calling redirty_page_for_writepage() but that
3159	 * would be ugly in the extreme.  So instead we would need to
3160	 * replicate parts of the code in the above functions,
3161	 * simplifying them because we wouldn't actually intend to
3162	 * write out the pages, but rather only collect contiguous
3163	 * logical block extents, call the multi-block allocator, and
3164	 * then update the buffer heads with the block allocations.
3165	 *
3166	 * For now, though, we'll cheat by calling filemap_flush(),
3167	 * which will map the blocks, and start the I/O, but not
3168	 * actually wait for the I/O to complete.
3169	 */
3170	return filemap_flush(inode->i_mapping);
3171}
3172
3173/*
3174 * bmap() is special.  It gets used by applications such as lilo and by
3175 * the swapper to find the on-disk block of a specific piece of data.
3176 *
3177 * Naturally, this is dangerous if the block concerned is still in the
3178 * journal.  If somebody makes a swapfile on an ext4 data-journaling
3179 * filesystem and enables swap, then they may get a nasty shock when the
3180 * data getting swapped to that swapfile suddenly gets overwritten by
3181 * the original zero's written out previously to the journal and
3182 * awaiting writeback in the kernel's buffer cache.
3183 *
3184 * So, if we see any bmap calls here on a modified, data-journaled file,
3185 * take extra steps to flush any blocks which might be in the cache.
3186 */
3187static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3188{
3189	struct inode *inode = mapping->host;
3190	journal_t *journal;
3191	int err;
3192
3193	/*
3194	 * We can get here for an inline file via the FIBMAP ioctl
3195	 */
3196	if (ext4_has_inline_data(inode))
3197		return 0;
3198
3199	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3200			test_opt(inode->i_sb, DELALLOC)) {
3201		/*
3202		 * With delalloc we want to sync the file
3203		 * so that we can make sure we allocate
3204		 * blocks for file
3205		 */
3206		filemap_write_and_wait(mapping);
3207	}
3208
3209	if (EXT4_JOURNAL(inode) &&
3210	    ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3211		/*
3212		 * This is a REALLY heavyweight approach, but the use of
3213		 * bmap on dirty files is expected to be extremely rare:
3214		 * only if we run lilo or swapon on a freshly made file
3215		 * do we expect this to happen.
3216		 *
3217		 * (bmap requires CAP_SYS_RAWIO so this does not
3218		 * represent an unprivileged user DOS attack --- we'd be
3219		 * in trouble if mortal users could trigger this path at
3220		 * will.)
3221		 *
3222		 * NB. EXT4_STATE_JDATA is not set on files other than
3223		 * regular files.  If somebody wants to bmap a directory
3224		 * or symlink and gets confused because the buffer
3225		 * hasn't yet been flushed to disk, they deserve
3226		 * everything they get.
3227		 */
3228
3229		ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3230		journal = EXT4_JOURNAL(inode);
3231		jbd2_journal_lock_updates(journal);
3232		err = jbd2_journal_flush(journal, 0);
3233		jbd2_journal_unlock_updates(journal);
3234
3235		if (err)
3236			return 0;
3237	}
3238
3239	return iomap_bmap(mapping, block, &ext4_iomap_ops);
3240}
3241
3242static int ext4_readpage(struct file *file, struct page *page)
3243{
3244	int ret = -EAGAIN;
3245	struct inode *inode = page->mapping->host;
3246
3247	trace_ext4_readpage(page);
3248
3249	if (ext4_has_inline_data(inode))
3250		ret = ext4_readpage_inline(inode, page);
3251
3252	if (ret == -EAGAIN)
3253		return ext4_mpage_readpages(inode, NULL, page);
3254
3255	return ret;
3256}
3257
3258static void ext4_readahead(struct readahead_control *rac)
3259{
3260	struct inode *inode = rac->mapping->host;
3261
3262	/* If the file has inline data, no need to do readahead. */
3263	if (ext4_has_inline_data(inode))
3264		return;
3265
3266	ext4_mpage_readpages(inode, rac, NULL);
3267}
3268
3269static void ext4_invalidatepage(struct page *page, unsigned int offset,
3270				unsigned int length)
3271{
3272	trace_ext4_invalidatepage(page, offset, length);
3273
3274	/* No journalling happens on data buffers when this function is used */
3275	WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3276
3277	block_invalidatepage(page, offset, length);
3278}
3279
3280static int __ext4_journalled_invalidatepage(struct page *page,
3281					    unsigned int offset,
3282					    unsigned int length)
3283{
3284	journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3285
3286	trace_ext4_journalled_invalidatepage(page, offset, length);
3287
3288	/*
3289	 * If it's a full truncate we just forget about the pending dirtying
3290	 */
3291	if (offset == 0 && length == PAGE_SIZE)
3292		ClearPageChecked(page);
3293
3294	return jbd2_journal_invalidatepage(journal, page, offset, length);
3295}
3296
3297/* Wrapper for aops... */
3298static void ext4_journalled_invalidatepage(struct page *page,
3299					   unsigned int offset,
3300					   unsigned int length)
3301{
3302	WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
3303}
3304
3305static int ext4_releasepage(struct page *page, gfp_t wait)
3306{
3307	journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3308
3309	trace_ext4_releasepage(page);
3310
3311	/* Page has dirty journalled data -> cannot release */
3312	if (PageChecked(page))
3313		return 0;
3314	if (journal)
3315		return jbd2_journal_try_to_free_buffers(journal, page);
3316	else
3317		return try_to_free_buffers(page);
3318}
3319
3320static bool ext4_inode_datasync_dirty(struct inode *inode)
3321{
3322	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3323
3324	if (journal) {
3325		if (jbd2_transaction_committed(journal,
3326			EXT4_I(inode)->i_datasync_tid))
3327			return false;
3328		if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3329			return !list_empty(&EXT4_I(inode)->i_fc_list);
3330		return true;
3331	}
3332
3333	/* Any metadata buffers to write? */
3334	if (!list_empty(&inode->i_mapping->private_list))
3335		return true;
3336	return inode->i_state & I_DIRTY_DATASYNC;
3337}
3338
3339static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3340			   struct ext4_map_blocks *map, loff_t offset,
3341			   loff_t length)
3342{
3343	u8 blkbits = inode->i_blkbits;
3344
3345	/*
3346	 * Writes that span EOF might trigger an I/O size update on completion,
3347	 * so consider them to be dirty for the purpose of O_DSYNC, even if
3348	 * there is no other metadata changes being made or are pending.
3349	 */
3350	iomap->flags = 0;
3351	if (ext4_inode_datasync_dirty(inode) ||
3352	    offset + length > i_size_read(inode))
3353		iomap->flags |= IOMAP_F_DIRTY;
3354
3355	if (map->m_flags & EXT4_MAP_NEW)
3356		iomap->flags |= IOMAP_F_NEW;
3357
3358	iomap->bdev = inode->i_sb->s_bdev;
3359	iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3360	iomap->offset = (u64) map->m_lblk << blkbits;
3361	iomap->length = (u64) map->m_len << blkbits;
3362
3363	if ((map->m_flags & EXT4_MAP_MAPPED) &&
3364	    !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3365		iomap->flags |= IOMAP_F_MERGED;
3366
3367	/*
3368	 * Flags passed to ext4_map_blocks() for direct I/O writes can result
3369	 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3370	 * set. In order for any allocated unwritten extents to be converted
3371	 * into written extents correctly within the ->end_io() handler, we
3372	 * need to ensure that the iomap->type is set appropriately. Hence, the
3373	 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3374	 * been set first.
3375	 */
3376	if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3377		iomap->type = IOMAP_UNWRITTEN;
3378		iomap->addr = (u64) map->m_pblk << blkbits;
3379	} else if (map->m_flags & EXT4_MAP_MAPPED) {
3380		iomap->type = IOMAP_MAPPED;
3381		iomap->addr = (u64) map->m_pblk << blkbits;
3382	} else {
3383		iomap->type = IOMAP_HOLE;
3384		iomap->addr = IOMAP_NULL_ADDR;
3385	}
3386}
3387
3388static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3389			    unsigned int flags)
3390{
3391	handle_t *handle;
3392	u8 blkbits = inode->i_blkbits;
3393	int ret, dio_credits, m_flags = 0, retries = 0;
3394
3395	/*
3396	 * Trim the mapping request to the maximum value that we can map at
3397	 * once for direct I/O.
3398	 */
3399	if (map->m_len > DIO_MAX_BLOCKS)
3400		map->m_len = DIO_MAX_BLOCKS;
3401	dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3402
3403retry:
3404	/*
3405	 * Either we allocate blocks and then don't get an unwritten extent, so
3406	 * in that case we have reserved enough credits. Or, the blocks are
3407	 * already allocated and unwritten. In that case, the extent conversion
3408	 * fits into the credits as well.
3409	 */
3410	handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3411	if (IS_ERR(handle))
3412		return PTR_ERR(handle);
3413
3414	/*
3415	 * DAX and direct I/O are the only two operations that are currently
3416	 * supported with IOMAP_WRITE.
3417	 */
3418	WARN_ON(!IS_DAX(inode) && !(flags & IOMAP_DIRECT));
3419	if (IS_DAX(inode))
3420		m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3421	/*
3422	 * We use i_size instead of i_disksize here because delalloc writeback
3423	 * can complete at any point during the I/O and subsequently push the
3424	 * i_disksize out to i_size. This could be beyond where direct I/O is
3425	 * happening and thus expose allocated blocks to direct I/O reads.
3426	 */
3427	else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))
3428		m_flags = EXT4_GET_BLOCKS_CREATE;
3429	else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3430		m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3431
3432	ret = ext4_map_blocks(handle, inode, map, m_flags);
3433
3434	/*
3435	 * We cannot fill holes in indirect tree based inodes as that could
3436	 * expose stale data in the case of a crash. Use the magic error code
3437	 * to fallback to buffered I/O.
3438	 */
3439	if (!m_flags && !ret)
3440		ret = -ENOTBLK;
3441
3442	ext4_journal_stop(handle);
3443	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3444		goto retry;
3445
3446	return ret;
3447}
3448
3449
3450static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3451		unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3452{
3453	int ret;
3454	struct ext4_map_blocks map;
3455	u8 blkbits = inode->i_blkbits;
3456
3457	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3458		return -EINVAL;
3459
3460	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3461		return -ERANGE;
3462
3463	/*
3464	 * Calculate the first and last logical blocks respectively.
3465	 */
3466	map.m_lblk = offset >> blkbits;
3467	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3468			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3469
3470	if (flags & IOMAP_WRITE) {
3471		/*
3472		 * We check here if the blocks are already allocated, then we
3473		 * don't need to start a journal txn and we can directly return
3474		 * the mapping information. This could boost performance
3475		 * especially in multi-threaded overwrite requests.
3476		 */
3477		if (offset + length <= i_size_read(inode)) {
3478			ret = ext4_map_blocks(NULL, inode, &map, 0);
3479			if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))
3480				goto out;
3481		}
3482		ret = ext4_iomap_alloc(inode, &map, flags);
3483	} else {
3484		ret = ext4_map_blocks(NULL, inode, &map, 0);
3485	}
3486
3487	if (ret < 0)
3488		return ret;
3489out:
3490	ext4_set_iomap(inode, iomap, &map, offset, length);
3491
3492	return 0;
3493}
3494
3495static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3496		loff_t length, unsigned flags, struct iomap *iomap,
3497		struct iomap *srcmap)
3498{
3499	int ret;
3500
3501	/*
3502	 * Even for writes we don't need to allocate blocks, so just pretend
3503	 * we are reading to save overhead of starting a transaction.
3504	 */
3505	flags &= ~IOMAP_WRITE;
3506	ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3507	WARN_ON_ONCE(iomap->type != IOMAP_MAPPED);
3508	return ret;
3509}
3510
3511static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3512			  ssize_t written, unsigned flags, struct iomap *iomap)
3513{
3514	/*
3515	 * Check to see whether an error occurred while writing out the data to
3516	 * the allocated blocks. If so, return the magic error code so that we
3517	 * fallback to buffered I/O and attempt to complete the remainder of
3518	 * the I/O. Any blocks that may have been allocated in preparation for
3519	 * the direct I/O will be reused during buffered I/O.
3520	 */
3521	if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)
3522		return -ENOTBLK;
3523
3524	return 0;
3525}
3526
3527const struct iomap_ops ext4_iomap_ops = {
3528	.iomap_begin		= ext4_iomap_begin,
3529	.iomap_end		= ext4_iomap_end,
3530};
3531
3532const struct iomap_ops ext4_iomap_overwrite_ops = {
3533	.iomap_begin		= ext4_iomap_overwrite_begin,
3534	.iomap_end		= ext4_iomap_end,
3535};
3536
3537static bool ext4_iomap_is_delalloc(struct inode *inode,
3538				   struct ext4_map_blocks *map)
3539{
3540	struct extent_status es;
3541	ext4_lblk_t offset = 0, end = map->m_lblk + map->m_len - 1;
3542
3543	ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
3544				  map->m_lblk, end, &es);
3545
3546	if (!es.es_len || es.es_lblk > end)
3547		return false;
3548
3549	if (es.es_lblk > map->m_lblk) {
3550		map->m_len = es.es_lblk - map->m_lblk;
3551		return false;
3552	}
3553
3554	offset = map->m_lblk - es.es_lblk;
3555	map->m_len = es.es_len - offset;
3556
3557	return true;
3558}
3559
3560static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3561				   loff_t length, unsigned int flags,
3562				   struct iomap *iomap, struct iomap *srcmap)
3563{
3564	int ret;
3565	bool delalloc = false;
3566	struct ext4_map_blocks map;
3567	u8 blkbits = inode->i_blkbits;
3568
3569	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3570		return -EINVAL;
3571
3572	if (ext4_has_inline_data(inode)) {
3573		ret = ext4_inline_data_iomap(inode, iomap);
3574		if (ret != -EAGAIN) {
3575			if (ret == 0 && offset >= iomap->length)
3576				ret = -ENOENT;
3577			return ret;
3578		}
3579	}
3580
3581	/*
3582	 * Calculate the first and last logical block respectively.
3583	 */
3584	map.m_lblk = offset >> blkbits;
3585	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3586			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3587
3588	/*
3589	 * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
3590	 * So handle it here itself instead of querying ext4_map_blocks().
3591	 * Since ext4_map_blocks() will warn about it and will return
3592	 * -EIO error.
3593	 */
3594	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
3595		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3596
3597		if (offset >= sbi->s_bitmap_maxbytes) {
3598			map.m_flags = 0;
3599			goto set_iomap;
3600		}
3601	}
3602
3603	ret = ext4_map_blocks(NULL, inode, &map, 0);
3604	if (ret < 0)
3605		return ret;
3606	if (ret == 0)
3607		delalloc = ext4_iomap_is_delalloc(inode, &map);
3608
3609set_iomap:
3610	ext4_set_iomap(inode, iomap, &map, offset, length);
3611	if (delalloc && iomap->type == IOMAP_HOLE)
3612		iomap->type = IOMAP_DELALLOC;
3613
3614	return 0;
3615}
3616
3617const struct iomap_ops ext4_iomap_report_ops = {
3618	.iomap_begin = ext4_iomap_begin_report,
3619};
3620
3621/*
3622 * Pages can be marked dirty completely asynchronously from ext4's journalling
3623 * activity.  By filemap_sync_pte(), try_to_unmap_one(), etc.  We cannot do
3624 * much here because ->set_page_dirty is called under VFS locks.  The page is
3625 * not necessarily locked.
3626 *
3627 * We cannot just dirty the page and leave attached buffers clean, because the
3628 * buffers' dirty state is "definitive".  We cannot just set the buffers dirty
3629 * or jbddirty because all the journalling code will explode.
3630 *
3631 * So what we do is to mark the page "pending dirty" and next time writepage
3632 * is called, propagate that into the buffers appropriately.
3633 */
3634static int ext4_journalled_set_page_dirty(struct page *page)
3635{
3636	SetPageChecked(page);
3637	return __set_page_dirty_nobuffers(page);
3638}
3639
3640static int ext4_set_page_dirty(struct page *page)
3641{
3642	WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
3643	WARN_ON_ONCE(!page_has_buffers(page));
3644	return __set_page_dirty_buffers(page);
3645}
3646
3647static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
3648				    struct file *file, sector_t *span)
3649{
3650	return iomap_swapfile_activate(sis, file, span,
3651				       &ext4_iomap_report_ops);
3652}
3653
3654static const struct address_space_operations ext4_aops = {
3655	.readpage		= ext4_readpage,
3656	.readahead		= ext4_readahead,
3657	.writepage		= ext4_writepage,
3658	.writepages		= ext4_writepages,
3659	.write_begin		= ext4_write_begin,
3660	.write_end		= ext4_write_end,
3661	.set_page_dirty		= ext4_set_page_dirty,
3662	.bmap			= ext4_bmap,
3663	.invalidatepage		= ext4_invalidatepage,
3664	.releasepage		= ext4_releasepage,
3665	.direct_IO		= noop_direct_IO,
3666	.migratepage		= buffer_migrate_page,
3667	.is_partially_uptodate  = block_is_partially_uptodate,
3668	.error_remove_page	= generic_error_remove_page,
3669	.swap_activate		= ext4_iomap_swap_activate,
3670};
3671
3672static const struct address_space_operations ext4_journalled_aops = {
3673	.readpage		= ext4_readpage,
3674	.readahead		= ext4_readahead,
3675	.writepage		= ext4_writepage,
3676	.writepages		= ext4_writepages,
3677	.write_begin		= ext4_write_begin,
3678	.write_end		= ext4_journalled_write_end,
3679	.set_page_dirty		= ext4_journalled_set_page_dirty,
3680	.bmap			= ext4_bmap,
3681	.invalidatepage		= ext4_journalled_invalidatepage,
3682	.releasepage		= ext4_releasepage,
3683	.direct_IO		= noop_direct_IO,
3684	.is_partially_uptodate  = block_is_partially_uptodate,
3685	.error_remove_page	= generic_error_remove_page,
3686	.swap_activate		= ext4_iomap_swap_activate,
3687};
3688
3689static const struct address_space_operations ext4_da_aops = {
3690	.readpage		= ext4_readpage,
3691	.readahead		= ext4_readahead,
3692	.writepage		= ext4_writepage,
3693	.writepages		= ext4_writepages,
3694	.write_begin		= ext4_da_write_begin,
3695	.write_end		= ext4_da_write_end,
3696	.set_page_dirty		= ext4_set_page_dirty,
3697	.bmap			= ext4_bmap,
3698	.invalidatepage		= ext4_invalidatepage,
3699	.releasepage		= ext4_releasepage,
3700	.direct_IO		= noop_direct_IO,
3701	.migratepage		= buffer_migrate_page,
3702	.is_partially_uptodate  = block_is_partially_uptodate,
3703	.error_remove_page	= generic_error_remove_page,
3704	.swap_activate		= ext4_iomap_swap_activate,
3705};
3706
3707static const struct address_space_operations ext4_dax_aops = {
3708	.writepages		= ext4_dax_writepages,
3709	.direct_IO		= noop_direct_IO,
3710	.set_page_dirty		= __set_page_dirty_no_writeback,
3711	.bmap			= ext4_bmap,
3712	.invalidatepage		= noop_invalidatepage,
3713	.swap_activate		= ext4_iomap_swap_activate,
3714};
3715
3716void ext4_set_aops(struct inode *inode)
3717{
3718	switch (ext4_inode_journal_mode(inode)) {
3719	case EXT4_INODE_ORDERED_DATA_MODE:
3720	case EXT4_INODE_WRITEBACK_DATA_MODE:
3721		break;
3722	case EXT4_INODE_JOURNAL_DATA_MODE:
3723		inode->i_mapping->a_ops = &ext4_journalled_aops;
3724		return;
3725	default:
3726		BUG();
3727	}
3728	if (IS_DAX(inode))
3729		inode->i_mapping->a_ops = &ext4_dax_aops;
3730	else if (test_opt(inode->i_sb, DELALLOC))
3731		inode->i_mapping->a_ops = &ext4_da_aops;
3732	else
3733		inode->i_mapping->a_ops = &ext4_aops;
3734}
3735
3736static int __ext4_block_zero_page_range(handle_t *handle,
3737		struct address_space *mapping, loff_t from, loff_t length)
3738{
3739	ext4_fsblk_t index = from >> PAGE_SHIFT;
3740	unsigned offset = from & (PAGE_SIZE-1);
3741	unsigned blocksize, pos;
3742	ext4_lblk_t iblock;
3743	struct inode *inode = mapping->host;
3744	struct buffer_head *bh;
3745	struct page *page;
3746	int err = 0;
3747
3748	page = find_or_create_page(mapping, from >> PAGE_SHIFT,
3749				   mapping_gfp_constraint(mapping, ~__GFP_FS));
3750	if (!page)
3751		return -ENOMEM;
3752
3753	blocksize = inode->i_sb->s_blocksize;
3754
3755	iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
3756
3757	if (!page_has_buffers(page))
3758		create_empty_buffers(page, blocksize, 0);
3759
3760	/* Find the buffer that contains "offset" */
3761	bh = page_buffers(page);
3762	pos = blocksize;
3763	while (offset >= pos) {
3764		bh = bh->b_this_page;
3765		iblock++;
3766		pos += blocksize;
3767	}
3768	if (buffer_freed(bh)) {
3769		BUFFER_TRACE(bh, "freed: skip");
3770		goto unlock;
3771	}
3772	if (!buffer_mapped(bh)) {
3773		BUFFER_TRACE(bh, "unmapped");
3774		ext4_get_block(inode, iblock, bh, 0);
3775		/* unmapped? It's a hole - nothing to do */
3776		if (!buffer_mapped(bh)) {
3777			BUFFER_TRACE(bh, "still unmapped");
3778			goto unlock;
3779		}
3780	}
3781
3782	/* Ok, it's mapped. Make sure it's up-to-date */
3783	if (PageUptodate(page))
3784		set_buffer_uptodate(bh);
3785
3786	if (!buffer_uptodate(bh)) {
3787		err = ext4_read_bh_lock(bh, 0, true);
3788		if (err)
3789			goto unlock;
3790		if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
3791			/* We expect the key to be set. */
3792			BUG_ON(!fscrypt_has_encryption_key(inode));
3793			err = fscrypt_decrypt_pagecache_blocks(page, blocksize,
3794							       bh_offset(bh));
3795			if (err) {
3796				clear_buffer_uptodate(bh);
3797				goto unlock;
3798			}
3799		}
3800	}
3801	if (ext4_should_journal_data(inode)) {
3802		BUFFER_TRACE(bh, "get write access");
3803		err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
3804						    EXT4_JTR_NONE);
3805		if (err)
3806			goto unlock;
3807	}
3808	zero_user(page, offset, length);
3809	BUFFER_TRACE(bh, "zeroed end of block");
3810
3811	if (ext4_should_journal_data(inode)) {
3812		err = ext4_handle_dirty_metadata(handle, inode, bh);
3813	} else {
3814		err = 0;
3815		mark_buffer_dirty(bh);
3816		if (ext4_should_order_data(inode))
3817			err = ext4_jbd2_inode_add_write(handle, inode, from,
3818					length);
3819	}
3820
3821unlock:
3822	unlock_page(page);
3823	put_page(page);
3824	return err;
3825}
3826
3827/*
3828 * ext4_block_zero_page_range() zeros out a mapping of length 'length'
3829 * starting from file offset 'from'.  The range to be zero'd must
3830 * be contained with in one block.  If the specified range exceeds
3831 * the end of the block it will be shortened to end of the block
3832 * that corresponds to 'from'
3833 */
3834static int ext4_block_zero_page_range(handle_t *handle,
3835		struct address_space *mapping, loff_t from, loff_t length)
3836{
3837	struct inode *inode = mapping->host;
3838	unsigned offset = from & (PAGE_SIZE-1);
3839	unsigned blocksize = inode->i_sb->s_blocksize;
3840	unsigned max = blocksize - (offset & (blocksize - 1));
3841
3842	/*
3843	 * correct length if it does not fall between
3844	 * 'from' and the end of the block
3845	 */
3846	if (length > max || length < 0)
3847		length = max;
3848
3849	if (IS_DAX(inode)) {
3850		return iomap_zero_range(inode, from, length, NULL,
3851					&ext4_iomap_ops);
3852	}
3853	return __ext4_block_zero_page_range(handle, mapping, from, length);
3854}
3855
3856/*
3857 * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3858 * up to the end of the block which corresponds to `from'.
3859 * This required during truncate. We need to physically zero the tail end
3860 * of that block so it doesn't yield old data if the file is later grown.
3861 */
3862static int ext4_block_truncate_page(handle_t *handle,
3863		struct address_space *mapping, loff_t from)
3864{
3865	unsigned offset = from & (PAGE_SIZE-1);
3866	unsigned length;
3867	unsigned blocksize;
3868	struct inode *inode = mapping->host;
3869
3870	/* If we are processing an encrypted inode during orphan list handling */
3871	if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
3872		return 0;
3873
3874	blocksize = inode->i_sb->s_blocksize;
3875	length = blocksize - (offset & (blocksize - 1));
3876
3877	return ext4_block_zero_page_range(handle, mapping, from, length);
3878}
3879
3880int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3881			     loff_t lstart, loff_t length)
3882{
3883	struct super_block *sb = inode->i_sb;
3884	struct address_space *mapping = inode->i_mapping;
3885	unsigned partial_start, partial_end;
3886	ext4_fsblk_t start, end;
3887	loff_t byte_end = (lstart + length - 1);
3888	int err = 0;
3889
3890	partial_start = lstart & (sb->s_blocksize - 1);
3891	partial_end = byte_end & (sb->s_blocksize - 1);
3892
3893	start = lstart >> sb->s_blocksize_bits;
3894	end = byte_end >> sb->s_blocksize_bits;
3895
3896	/* Handle partial zero within the single block */
3897	if (start == end &&
3898	    (partial_start || (partial_end != sb->s_blocksize - 1))) {
3899		err = ext4_block_zero_page_range(handle, mapping,
3900						 lstart, length);
3901		return err;
3902	}
3903	/* Handle partial zero out on the start of the range */
3904	if (partial_start) {
3905		err = ext4_block_zero_page_range(handle, mapping,
3906						 lstart, sb->s_blocksize);
3907		if (err)
3908			return err;
3909	}
3910	/* Handle partial zero out on the end of the range */
3911	if (partial_end != sb->s_blocksize - 1)
3912		err = ext4_block_zero_page_range(handle, mapping,
3913						 byte_end - partial_end,
3914						 partial_end + 1);
3915	return err;
3916}
3917
3918int ext4_can_truncate(struct inode *inode)
3919{
3920	if (S_ISREG(inode->i_mode))
3921		return 1;
3922	if (S_ISDIR(inode->i_mode))
3923		return 1;
3924	if (S_ISLNK(inode->i_mode))
3925		return !ext4_inode_is_fast_symlink(inode);
3926	return 0;
3927}
3928
3929/*
3930 * We have to make sure i_disksize gets properly updated before we truncate
3931 * page cache due to hole punching or zero range. Otherwise i_disksize update
3932 * can get lost as it may have been postponed to submission of writeback but
3933 * that will never happen after we truncate page cache.
3934 */
3935int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3936				      loff_t len)
3937{
3938	handle_t *handle;
3939	int ret;
3940
3941	loff_t size = i_size_read(inode);
3942
3943	WARN_ON(!inode_is_locked(inode));
3944	if (offset > size || offset + len < size)
3945		return 0;
3946
3947	if (EXT4_I(inode)->i_disksize >= size)
3948		return 0;
3949
3950	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3951	if (IS_ERR(handle))
3952		return PTR_ERR(handle);
3953	ext4_update_i_disksize(inode, size);
3954	ret = ext4_mark_inode_dirty(handle, inode);
3955	ext4_journal_stop(handle);
3956
3957	return ret;
3958}
3959
3960static void ext4_wait_dax_page(struct ext4_inode_info *ei)
3961{
3962	up_write(&ei->i_mmap_sem);
3963	schedule();
3964	down_write(&ei->i_mmap_sem);
3965}
3966
3967int ext4_break_layouts(struct inode *inode)
3968{
3969	struct ext4_inode_info *ei = EXT4_I(inode);
3970	struct page *page;
3971	int error;
3972
3973	if (WARN_ON_ONCE(!rwsem_is_locked(&ei->i_mmap_sem)))
3974		return -EINVAL;
3975
3976	do {
3977		page = dax_layout_busy_page(inode->i_mapping);
3978		if (!page)
3979			return 0;
3980
3981		error = ___wait_var_event(&page->_refcount,
3982				atomic_read(&page->_refcount) == 1,
3983				TASK_INTERRUPTIBLE, 0, 0,
3984				ext4_wait_dax_page(ei));
3985	} while (error == 0);
3986
3987	return error;
3988}
3989
3990/*
3991 * ext4_punch_hole: punches a hole in a file by releasing the blocks
3992 * associated with the given offset and length
3993 *
3994 * @inode:  File inode
3995 * @offset: The offset where the hole will begin
3996 * @len:    The length of the hole
3997 *
3998 * Returns: 0 on success or negative on failure
3999 */
4000
4001int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
4002{
4003	struct super_block *sb = inode->i_sb;
4004	ext4_lblk_t first_block, stop_block;
4005	struct address_space *mapping = inode->i_mapping;
4006	loff_t first_block_offset, last_block_offset;
4007	handle_t *handle;
4008	unsigned int credits;
4009	int ret = 0, ret2 = 0;
4010
4011	trace_ext4_punch_hole(inode, offset, length, 0);
4012
4013	ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
4014	if (ext4_has_inline_data(inode)) {
4015		down_write(&EXT4_I(inode)->i_mmap_sem);
4016		ret = ext4_convert_inline_data(inode);
4017		up_write(&EXT4_I(inode)->i_mmap_sem);
4018		if (ret)
4019			return ret;
4020	}
4021
4022	/*
4023	 * Write out all dirty pages to avoid race conditions
4024	 * Then release them.
4025	 */
4026	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
4027		ret = filemap_write_and_wait_range(mapping, offset,
4028						   offset + length - 1);
4029		if (ret)
4030			return ret;
4031	}
4032
4033	inode_lock(inode);
4034
4035	/* No need to punch hole beyond i_size */
4036	if (offset >= inode->i_size)
4037		goto out_mutex;
4038
4039	/*
4040	 * If the hole extends beyond i_size, set the hole
4041	 * to end after the page that contains i_size
4042	 */
4043	if (offset + length > inode->i_size) {
4044		length = inode->i_size +
4045		   PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
4046		   offset;
4047	}
4048
4049	if (offset & (sb->s_blocksize - 1) ||
4050	    (offset + length) & (sb->s_blocksize - 1)) {
4051		/*
4052		 * Attach jinode to inode for jbd2 if we do any zeroing of
4053		 * partial block
4054		 */
4055		ret = ext4_inode_attach_jinode(inode);
4056		if (ret < 0)
4057			goto out_mutex;
4058
4059	}
4060
4061	/* Wait all existing dio workers, newcomers will block on i_mutex */
4062	inode_dio_wait(inode);
4063
4064	/*
4065	 * Prevent page faults from reinstantiating pages we have released from
4066	 * page cache.
4067	 */
4068	down_write(&EXT4_I(inode)->i_mmap_sem);
4069
4070	ret = ext4_break_layouts(inode);
4071	if (ret)
4072		goto out_dio;
4073
4074	first_block_offset = round_up(offset, sb->s_blocksize);
4075	last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
4076
4077	/* Now release the pages and zero block aligned part of pages*/
4078	if (last_block_offset > first_block_offset) {
4079		ret = ext4_update_disksize_before_punch(inode, offset, length);
4080		if (ret)
4081			goto out_dio;
4082		truncate_pagecache_range(inode, first_block_offset,
4083					 last_block_offset);
4084	}
4085
4086	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4087		credits = ext4_writepage_trans_blocks(inode);
4088	else
4089		credits = ext4_blocks_for_truncate(inode);
4090	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4091	if (IS_ERR(handle)) {
4092		ret = PTR_ERR(handle);
4093		ext4_std_error(sb, ret);
4094		goto out_dio;
4095	}
4096
4097	ret = ext4_zero_partial_blocks(handle, inode, offset,
4098				       length);
4099	if (ret)
4100		goto out_stop;
4101
4102	first_block = (offset + sb->s_blocksize - 1) >>
4103		EXT4_BLOCK_SIZE_BITS(sb);
4104	stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4105
4106	/* If there are blocks to remove, do it */
4107	if (stop_block > first_block) {
4108
4109		down_write(&EXT4_I(inode)->i_data_sem);
4110		ext4_discard_preallocations(inode, 0);
4111
4112		ret = ext4_es_remove_extent(inode, first_block,
4113					    stop_block - first_block);
4114		if (ret) {
4115			up_write(&EXT4_I(inode)->i_data_sem);
4116			goto out_stop;
4117		}
4118
4119		if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4120			ret = ext4_ext_remove_space(inode, first_block,
4121						    stop_block - 1);
4122		else
4123			ret = ext4_ind_remove_space(handle, inode, first_block,
4124						    stop_block);
4125
4126		up_write(&EXT4_I(inode)->i_data_sem);
4127	}
4128	ext4_fc_track_range(handle, inode, first_block, stop_block);
4129	if (IS_SYNC(inode))
4130		ext4_handle_sync(handle);
4131
4132	inode->i_mtime = inode->i_ctime = current_time(inode);
4133	ret2 = ext4_mark_inode_dirty(handle, inode);
4134	if (unlikely(ret2))
4135		ret = ret2;
4136	if (ret >= 0)
4137		ext4_update_inode_fsync_trans(handle, inode, 1);
4138out_stop:
4139	ext4_journal_stop(handle);
4140out_dio:
4141	up_write(&EXT4_I(inode)->i_mmap_sem);
4142out_mutex:
4143	inode_unlock(inode);
4144	return ret;
4145}
4146
4147int ext4_inode_attach_jinode(struct inode *inode)
4148{
4149	struct ext4_inode_info *ei = EXT4_I(inode);
4150	struct jbd2_inode *jinode;
4151
4152	if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4153		return 0;
4154
4155	jinode = jbd2_alloc_inode(GFP_KERNEL);
4156	spin_lock(&inode->i_lock);
4157	if (!ei->jinode) {
4158		if (!jinode) {
4159			spin_unlock(&inode->i_lock);
4160			return -ENOMEM;
4161		}
4162		ei->jinode = jinode;
4163		jbd2_journal_init_jbd_inode(ei->jinode, inode);
4164		jinode = NULL;
4165	}
4166	spin_unlock(&inode->i_lock);
4167	if (unlikely(jinode != NULL))
4168		jbd2_free_inode(jinode);
4169	return 0;
4170}
4171
4172/*
4173 * ext4_truncate()
4174 *
4175 * We block out ext4_get_block() block instantiations across the entire
4176 * transaction, and VFS/VM ensures that ext4_truncate() cannot run
4177 * simultaneously on behalf of the same inode.
4178 *
4179 * As we work through the truncate and commit bits of it to the journal there
4180 * is one core, guiding principle: the file's tree must always be consistent on
4181 * disk.  We must be able to restart the truncate after a crash.
4182 *
4183 * The file's tree may be transiently inconsistent in memory (although it
4184 * probably isn't), but whenever we close off and commit a journal transaction,
4185 * the contents of (the filesystem + the journal) must be consistent and
4186 * restartable.  It's pretty simple, really: bottom up, right to left (although
4187 * left-to-right works OK too).
4188 *
4189 * Note that at recovery time, journal replay occurs *before* the restart of
4190 * truncate against the orphan inode list.
4191 *
4192 * The committed inode has the new, desired i_size (which is the same as
4193 * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see
4194 * that this inode's truncate did not complete and it will again call
4195 * ext4_truncate() to have another go.  So there will be instantiated blocks
4196 * to the right of the truncation point in a crashed ext4 filesystem.  But
4197 * that's fine - as long as they are linked from the inode, the post-crash
4198 * ext4_truncate() run will find them and release them.
4199 */
4200int ext4_truncate(struct inode *inode)
4201{
4202	struct ext4_inode_info *ei = EXT4_I(inode);
4203	unsigned int credits;
4204	int err = 0, err2;
4205	handle_t *handle;
4206	struct address_space *mapping = inode->i_mapping;
4207
4208	/*
4209	 * There is a possibility that we're either freeing the inode
4210	 * or it's a completely new inode. In those cases we might not
4211	 * have i_mutex locked because it's not necessary.
4212	 */
4213	if (!(inode->i_state & (I_NEW|I_FREEING)))
4214		WARN_ON(!inode_is_locked(inode));
4215	trace_ext4_truncate_enter(inode);
4216
4217	if (!ext4_can_truncate(inode))
4218		goto out_trace;
4219
4220	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4221		ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4222
4223	if (ext4_has_inline_data(inode)) {
4224		int has_inline = 1;
4225
4226		err = ext4_inline_data_truncate(inode, &has_inline);
4227		if (err || has_inline)
4228			goto out_trace;
4229	}
4230
4231	/* If we zero-out tail of the page, we have to create jinode for jbd2 */
4232	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4233		if (ext4_inode_attach_jinode(inode) < 0)
4234			goto out_trace;
4235	}
4236
4237	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4238		credits = ext4_writepage_trans_blocks(inode);
4239	else
4240		credits = ext4_blocks_for_truncate(inode);
4241
4242	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4243	if (IS_ERR(handle)) {
4244		err = PTR_ERR(handle);
4245		goto out_trace;
4246	}
4247
4248	if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4249		ext4_block_truncate_page(handle, mapping, inode->i_size);
4250
4251	/*
4252	 * We add the inode to the orphan list, so that if this
4253	 * truncate spans multiple transactions, and we crash, we will
4254	 * resume the truncate when the filesystem recovers.  It also
4255	 * marks the inode dirty, to catch the new size.
4256	 *
4257	 * Implication: the file must always be in a sane, consistent
4258	 * truncatable state while each transaction commits.
4259	 */
4260	err = ext4_orphan_add(handle, inode);
4261	if (err)
4262		goto out_stop;
4263
4264	down_write(&EXT4_I(inode)->i_data_sem);
4265
4266	ext4_discard_preallocations(inode, 0);
4267
4268	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4269		err = ext4_ext_truncate(handle, inode);
4270	else
4271		ext4_ind_truncate(handle, inode);
4272
4273	up_write(&ei->i_data_sem);
4274	if (err)
4275		goto out_stop;
4276
4277	if (IS_SYNC(inode))
4278		ext4_handle_sync(handle);
4279
4280out_stop:
4281	/*
4282	 * If this was a simple ftruncate() and the file will remain alive,
4283	 * then we need to clear up the orphan record which we created above.
4284	 * However, if this was a real unlink then we were called by
4285	 * ext4_evict_inode(), and we allow that function to clean up the
4286	 * orphan info for us.
4287	 */
4288	if (inode->i_nlink)
4289		ext4_orphan_del(handle, inode);
4290
4291	inode->i_mtime = inode->i_ctime = current_time(inode);
4292	err2 = ext4_mark_inode_dirty(handle, inode);
4293	if (unlikely(err2 && !err))
4294		err = err2;
4295	ext4_journal_stop(handle);
4296
4297out_trace:
4298	trace_ext4_truncate_exit(inode);
4299	return err;
4300}
4301
4302/*
4303 * ext4_get_inode_loc returns with an extra refcount against the inode's
4304 * underlying buffer_head on success. If 'in_mem' is true, we have all
4305 * data in memory that is needed to recreate the on-disk version of this
4306 * inode.
4307 */
4308static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4309				struct ext4_iloc *iloc, int in_mem,
4310				ext4_fsblk_t *ret_block)
4311{
4312	struct ext4_group_desc	*gdp;
4313	struct buffer_head	*bh;
4314	ext4_fsblk_t		block;
4315	struct blk_plug		plug;
4316	int			inodes_per_block, inode_offset;
4317
4318	iloc->bh = NULL;
4319	if (ino < EXT4_ROOT_INO ||
4320	    ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
4321		return -EFSCORRUPTED;
4322
4323	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
4324	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4325	if (!gdp)
4326		return -EIO;
4327
4328	/*
4329	 * Figure out the offset within the block group inode table
4330	 */
4331	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4332	inode_offset = ((ino - 1) %
4333			EXT4_INODES_PER_GROUP(sb));
4334	block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
4335	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4336
4337	bh = sb_getblk(sb, block);
4338	if (unlikely(!bh))
4339		return -ENOMEM;
4340	if (ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO))
4341		goto simulate_eio;
4342	if (!buffer_uptodate(bh)) {
4343		lock_buffer(bh);
4344
4345		if (ext4_buffer_uptodate(bh)) {
4346			/* someone brought it uptodate while we waited */
4347			unlock_buffer(bh);
4348			goto has_buffer;
4349		}
4350
4351		/*
4352		 * If we have all information of the inode in memory and this
4353		 * is the only valid inode in the block, we need not read the
4354		 * block.
4355		 */
4356		if (in_mem) {
4357			struct buffer_head *bitmap_bh;
4358			int i, start;
4359
4360			start = inode_offset & ~(inodes_per_block - 1);
4361
4362			/* Is the inode bitmap in cache? */
4363			bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
4364			if (unlikely(!bitmap_bh))
4365				goto make_io;
4366
4367			/*
4368			 * If the inode bitmap isn't in cache then the
4369			 * optimisation may end up performing two reads instead
4370			 * of one, so skip it.
4371			 */
4372			if (!buffer_uptodate(bitmap_bh)) {
4373				brelse(bitmap_bh);
4374				goto make_io;
4375			}
4376			for (i = start; i < start + inodes_per_block; i++) {
4377				if (i == inode_offset)
4378					continue;
4379				if (ext4_test_bit(i, bitmap_bh->b_data))
4380					break;
4381			}
4382			brelse(bitmap_bh);
4383			if (i == start + inodes_per_block) {
4384				/* all other inodes are free, so skip I/O */
4385				memset(bh->b_data, 0, bh->b_size);
4386				set_buffer_uptodate(bh);
4387				unlock_buffer(bh);
4388				goto has_buffer;
4389			}
4390		}
4391
4392make_io:
4393		/*
4394		 * If we need to do any I/O, try to pre-readahead extra
4395		 * blocks from the inode table.
4396		 */
4397		blk_start_plug(&plug);
4398		if (EXT4_SB(sb)->s_inode_readahead_blks) {
4399			ext4_fsblk_t b, end, table;
4400			unsigned num;
4401			__u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4402
4403			table = ext4_inode_table(sb, gdp);
4404			/* s_inode_readahead_blks is always a power of 2 */
4405			b = block & ~((ext4_fsblk_t) ra_blks - 1);
4406			if (table > b)
4407				b = table;
4408			end = b + ra_blks;
4409			num = EXT4_INODES_PER_GROUP(sb);
4410			if (ext4_has_group_desc_csum(sb))
4411				num -= ext4_itable_unused_count(sb, gdp);
4412			table += num / inodes_per_block;
4413			if (end > table)
4414				end = table;
4415			while (b <= end)
4416				ext4_sb_breadahead_unmovable(sb, b++);
4417		}
4418
4419		/*
4420		 * There are other valid inodes in the buffer, this inode
4421		 * has in-inode xattrs, or we don't have this inode in memory.
4422		 * Read the block from disk.
4423		 */
4424		trace_ext4_load_inode(sb, ino);
4425		ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);
4426		blk_finish_plug(&plug);
4427		wait_on_buffer(bh);
4428		if (!buffer_uptodate(bh)) {
4429		simulate_eio:
4430			if (ret_block)
4431				*ret_block = block;
4432			brelse(bh);
4433			return -EIO;
4434		}
4435	}
4436has_buffer:
4437	iloc->bh = bh;
4438	return 0;
4439}
4440
4441static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4442					struct ext4_iloc *iloc)
4443{
4444	ext4_fsblk_t err_blk;
4445	int ret;
4446
4447	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc, 0,
4448					&err_blk);
4449
4450	if (ret == -EIO)
4451		ext4_error_inode_block(inode, err_blk, EIO,
4452					"unable to read itable block");
4453
4454	return ret;
4455}
4456
4457int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4458{
4459	ext4_fsblk_t err_blk;
4460	int ret;
4461
4462	/* We have all inode data except xattrs in memory here. */
4463	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc,
4464		!ext4_test_inode_state(inode, EXT4_STATE_XATTR), &err_blk);
4465
4466	if (ret == -EIO)
4467		ext4_error_inode_block(inode, err_blk, EIO,
4468					"unable to read itable block");
4469
4470	return ret;
4471}
4472
4473
4474int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
4475			  struct ext4_iloc *iloc)
4476{
4477	return __ext4_get_inode_loc(sb, ino, iloc, 0, NULL);
4478}
4479
4480static bool ext4_should_enable_dax(struct inode *inode)
4481{
4482	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4483
4484	if (test_opt2(inode->i_sb, DAX_NEVER))
4485		return false;
4486	if (!S_ISREG(inode->i_mode))
4487		return false;
4488	if (ext4_should_journal_data(inode))
4489		return false;
4490	if (ext4_has_inline_data(inode))
4491		return false;
4492	if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
4493		return false;
4494	if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
4495		return false;
4496	if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
4497		return false;
4498	if (test_opt(inode->i_sb, DAX_ALWAYS))
4499		return true;
4500
4501	return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
4502}
4503
4504void ext4_set_inode_flags(struct inode *inode, bool init)
4505{
4506	unsigned int flags = EXT4_I(inode)->i_flags;
4507	unsigned int new_fl = 0;
4508
4509	WARN_ON_ONCE(IS_DAX(inode) && init);
4510
4511	if (flags & EXT4_SYNC_FL)
4512		new_fl |= S_SYNC;
4513	if (flags & EXT4_APPEND_FL)
4514		new_fl |= S_APPEND;
4515	if (flags & EXT4_IMMUTABLE_FL)
4516		new_fl |= S_IMMUTABLE;
4517	if (flags & EXT4_NOATIME_FL)
4518		new_fl |= S_NOATIME;
4519	if (flags & EXT4_DIRSYNC_FL)
4520		new_fl |= S_DIRSYNC;
4521
4522	/* Because of the way inode_set_flags() works we must preserve S_DAX
4523	 * here if already set. */
4524	new_fl |= (inode->i_flags & S_DAX);
4525	if (init && ext4_should_enable_dax(inode))
4526		new_fl |= S_DAX;
4527
4528	if (flags & EXT4_ENCRYPT_FL)
4529		new_fl |= S_ENCRYPTED;
4530	if (flags & EXT4_CASEFOLD_FL)
4531		new_fl |= S_CASEFOLD;
4532	if (flags & EXT4_VERITY_FL)
4533		new_fl |= S_VERITY;
4534	inode_set_flags(inode, new_fl,
4535			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
4536			S_ENCRYPTED|S_CASEFOLD|S_VERITY);
4537}
4538
4539static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
4540				  struct ext4_inode_info *ei)
4541{
4542	blkcnt_t i_blocks ;
4543	struct inode *inode = &(ei->vfs_inode);
4544	struct super_block *sb = inode->i_sb;
4545
4546	if (ext4_has_feature_huge_file(sb)) {
4547		/* we are using combined 48 bit field */
4548		i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4549					le32_to_cpu(raw_inode->i_blocks_lo);
4550		if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
4551			/* i_blocks represent file system block size */
4552			return i_blocks  << (inode->i_blkbits - 9);
4553		} else {
4554			return i_blocks;
4555		}
4556	} else {
4557		return le32_to_cpu(raw_inode->i_blocks_lo);
4558	}
4559}
4560
4561static inline int ext4_iget_extra_inode(struct inode *inode,
4562					 struct ext4_inode *raw_inode,
4563					 struct ext4_inode_info *ei)
4564{
4565	__le32 *magic = (void *)raw_inode +
4566			EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
4567
4568	if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
4569	    EXT4_INODE_SIZE(inode->i_sb) &&
4570	    *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
4571		ext4_set_inode_state(inode, EXT4_STATE_XATTR);
4572		return ext4_find_inline_data_nolock(inode);
4573	} else
4574		EXT4_I(inode)->i_inline_off = 0;
4575	return 0;
4576}
4577
4578int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4579{
4580	if (!ext4_has_feature_project(inode->i_sb))
4581		return -EOPNOTSUPP;
4582	*projid = EXT4_I(inode)->i_projid;
4583	return 0;
4584}
4585
4586/*
4587 * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
4588 * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
4589 * set.
4590 */
4591static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
4592{
4593	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4594		inode_set_iversion_raw(inode, val);
4595	else
4596		inode_set_iversion_queried(inode, val);
4597}
4598static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4599{
4600	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4601		return inode_peek_iversion_raw(inode);
4602	else
4603		return inode_peek_iversion(inode);
4604}
4605
4606struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
4607			  ext4_iget_flags flags, const char *function,
4608			  unsigned int line)
4609{
4610	struct ext4_iloc iloc;
4611	struct ext4_inode *raw_inode;
4612	struct ext4_inode_info *ei;
4613	struct ext4_super_block *es = EXT4_SB(sb)->s_es;
4614	struct inode *inode;
4615	journal_t *journal = EXT4_SB(sb)->s_journal;
4616	long ret;
4617	loff_t size;
4618	int block;
4619	uid_t i_uid;
4620	gid_t i_gid;
4621	projid_t i_projid;
4622
4623	if ((!(flags & EXT4_IGET_SPECIAL) &&
4624	     ((ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) ||
4625	      ino == le32_to_cpu(es->s_usr_quota_inum) ||
4626	      ino == le32_to_cpu(es->s_grp_quota_inum) ||
4627	      ino == le32_to_cpu(es->s_prj_quota_inum))) ||
4628	    (ino < EXT4_ROOT_INO) ||
4629	    (ino > le32_to_cpu(es->s_inodes_count))) {
4630		if (flags & EXT4_IGET_HANDLE)
4631			return ERR_PTR(-ESTALE);
4632		__ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
4633			     "inode #%lu: comm %s: iget: illegal inode #",
4634			     ino, current->comm);
4635		return ERR_PTR(-EFSCORRUPTED);
4636	}
4637
4638	inode = iget_locked(sb, ino);
4639	if (!inode)
4640		return ERR_PTR(-ENOMEM);
4641	if (!(inode->i_state & I_NEW))
4642		return inode;
4643
4644	ei = EXT4_I(inode);
4645	iloc.bh = NULL;
4646
4647	ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
4648	if (ret < 0)
4649		goto bad_inode;
4650	raw_inode = ext4_raw_inode(&iloc);
4651
4652	if ((ino == EXT4_ROOT_INO) && (raw_inode->i_links_count == 0)) {
4653		ext4_error_inode(inode, function, line, 0,
4654				 "iget: root inode unallocated");
4655		ret = -EFSCORRUPTED;
4656		goto bad_inode;
4657	}
4658
4659	if ((flags & EXT4_IGET_HANDLE) &&
4660	    (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
4661		ret = -ESTALE;
4662		goto bad_inode;
4663	}
4664
4665	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4666		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4667		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4668			EXT4_INODE_SIZE(inode->i_sb) ||
4669		    (ei->i_extra_isize & 3)) {
4670			ext4_error_inode(inode, function, line, 0,
4671					 "iget: bad extra_isize %u "
4672					 "(inode size %u)",
4673					 ei->i_extra_isize,
4674					 EXT4_INODE_SIZE(inode->i_sb));
4675			ret = -EFSCORRUPTED;
4676			goto bad_inode;
4677		}
4678	} else
4679		ei->i_extra_isize = 0;
4680
4681	/* Precompute checksum seed for inode metadata */
4682	if (ext4_has_metadata_csum(sb)) {
4683		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4684		__u32 csum;
4685		__le32 inum = cpu_to_le32(inode->i_ino);
4686		__le32 gen = raw_inode->i_generation;
4687		csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4688				   sizeof(inum));
4689		ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4690					      sizeof(gen));
4691	}
4692
4693	if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
4694	    ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
4695	     (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
4696		ext4_error_inode_err(inode, function, line, 0,
4697				EFSBADCRC, "iget: checksum invalid");
4698		ret = -EFSBADCRC;
4699		goto bad_inode;
4700	}
4701
4702	inode->i_mode = le16_to_cpu(raw_inode->i_mode);
4703	i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4704	i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
4705	if (ext4_has_feature_project(sb) &&
4706	    EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4707	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4708		i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4709	else
4710		i_projid = EXT4_DEF_PROJID;
4711
4712	if (!(test_opt(inode->i_sb, NO_UID32))) {
4713		i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4714		i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
4715	}
4716	i_uid_write(inode, i_uid);
4717	i_gid_write(inode, i_gid);
4718	ei->i_projid = make_kprojid(&init_user_ns, i_projid);
4719	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
4720
4721	ext4_clear_state_flags(ei);	/* Only relevant on 32-bit archs */
4722	ei->i_inline_off = 0;
4723	ei->i_dir_start_lookup = 0;
4724	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4725	/* We now have enough fields to check if the inode was active or not.
4726	 * This is needed because nfsd might try to access dead inodes
4727	 * the test is that same one that e2fsck uses
4728	 * NeilBrown 1999oct15
4729	 */
4730	if (inode->i_nlink == 0) {
4731		if ((inode->i_mode == 0 ||
4732		     !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4733		    ino != EXT4_BOOT_LOADER_INO) {
4734			/* this inode is deleted */
4735			ret = -ESTALE;
4736			goto bad_inode;
4737		}
4738		/* The only unlinked inodes we let through here have
4739		 * valid i_mode and are being read by the orphan
4740		 * recovery code: that's fine, we're about to complete
4741		 * the process of deleting those.
4742		 * OR it is the EXT4_BOOT_LOADER_INO which is
4743		 * not initialized on a new filesystem. */
4744	}
4745	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
4746	ext4_set_inode_flags(inode, true);
4747	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
4748	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
4749	if (ext4_has_feature_64bit(sb))
4750		ei->i_file_acl |=
4751			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
4752	inode->i_size = ext4_isize(sb, raw_inode);
4753	if ((size = i_size_read(inode)) < 0) {
4754		ext4_error_inode(inode, function, line, 0,
4755				 "iget: bad i_size value: %lld", size);
4756		ret = -EFSCORRUPTED;
4757		goto bad_inode;
4758	}
4759	/*
4760	 * If dir_index is not enabled but there's dir with INDEX flag set,
4761	 * we'd normally treat htree data as empty space. But with metadata
4762	 * checksumming that corrupts checksums so forbid that.
4763	 */
4764	if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
4765	    ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
4766		ext4_error_inode(inode, function, line, 0,
4767			 "iget: Dir with htree data on filesystem without dir_index feature.");
4768		ret = -EFSCORRUPTED;
4769		goto bad_inode;
4770	}
4771	ei->i_disksize = inode->i_size;
4772#ifdef CONFIG_QUOTA
4773	ei->i_reserved_quota = 0;
4774#endif
4775	inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4776	ei->i_block_group = iloc.block_group;
4777	ei->i_last_alloc_group = ~0;
4778	/*
4779	 * NOTE! The in-memory inode i_data array is in little-endian order
4780	 * even on big-endian machines: we do NOT byteswap the block numbers!
4781	 */
4782	for (block = 0; block < EXT4_N_BLOCKS; block++)
4783		ei->i_data[block] = raw_inode->i_block[block];
4784	INIT_LIST_HEAD(&ei->i_orphan);
4785	ext4_fc_init_inode(&ei->vfs_inode);
4786
4787	/*
4788	 * Set transaction id's of transactions that have to be committed
4789	 * to finish f[data]sync. We set them to currently running transaction
4790	 * as we cannot be sure that the inode or some of its metadata isn't
4791	 * part of the transaction - the inode could have been reclaimed and
4792	 * now it is reread from disk.
4793	 */
4794	if (journal) {
4795		transaction_t *transaction;
4796		tid_t tid;
4797
4798		read_lock(&journal->j_state_lock);
4799		if (journal->j_running_transaction)
4800			transaction = journal->j_running_transaction;
4801		else
4802			transaction = journal->j_committing_transaction;
4803		if (transaction)
4804			tid = transaction->t_tid;
4805		else
4806			tid = journal->j_commit_sequence;
4807		read_unlock(&journal->j_state_lock);
4808		ei->i_sync_tid = tid;
4809		ei->i_datasync_tid = tid;
4810	}
4811
4812	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4813		if (ei->i_extra_isize == 0) {
4814			/* The extra space is currently unused. Use it. */
4815			BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
4816			ei->i_extra_isize = sizeof(struct ext4_inode) -
4817					    EXT4_GOOD_OLD_INODE_SIZE;
4818		} else {
4819			ret = ext4_iget_extra_inode(inode, raw_inode, ei);
4820			if (ret)
4821				goto bad_inode;
4822		}
4823	}
4824
4825	EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
4826	EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
4827	EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
4828	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4829
4830	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4831		u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
4832
4833		if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4834			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4835				ivers |=
4836		    (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4837		}
4838		ext4_inode_set_iversion_queried(inode, ivers);
4839	}
4840
4841	ret = 0;
4842	if (ei->i_file_acl &&
4843	    !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
4844		ext4_error_inode(inode, function, line, 0,
4845				 "iget: bad extended attribute block %llu",
4846				 ei->i_file_acl);
4847		ret = -EFSCORRUPTED;
4848		goto bad_inode;
4849	} else if (!ext4_has_inline_data(inode)) {
4850		/* validate the block references in the inode */
4851		if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
4852			(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4853			(S_ISLNK(inode->i_mode) &&
4854			!ext4_inode_is_fast_symlink(inode)))) {
4855			if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4856				ret = ext4_ext_check_inode(inode);
4857			else
4858				ret = ext4_ind_check_inode(inode);
4859		}
4860	}
4861	if (ret)
4862		goto bad_inode;
4863
4864	if (S_ISREG(inode->i_mode)) {
4865		inode->i_op = &ext4_file_inode_operations;
4866		inode->i_fop = &ext4_file_operations;
4867		ext4_set_aops(inode);
4868	} else if (S_ISDIR(inode->i_mode)) {
4869		inode->i_op = &ext4_dir_inode_operations;
4870		inode->i_fop = &ext4_dir_operations;
4871	} else if (S_ISLNK(inode->i_mode)) {
4872		/* VFS does not allow setting these so must be corruption */
4873		if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
4874			ext4_error_inode(inode, function, line, 0,
4875					 "iget: immutable or append flags "
4876					 "not allowed on symlinks");
4877			ret = -EFSCORRUPTED;
4878			goto bad_inode;
4879		}
4880		if (IS_ENCRYPTED(inode)) {
4881			inode->i_op = &ext4_encrypted_symlink_inode_operations;
4882			ext4_set_aops(inode);
4883		} else if (ext4_inode_is_fast_symlink(inode)) {
4884			inode->i_link = (char *)ei->i_data;
4885			inode->i_op = &ext4_fast_symlink_inode_operations;
4886			nd_terminate_link(ei->i_data, inode->i_size,
4887				sizeof(ei->i_data) - 1);
4888		} else {
4889			inode->i_op = &ext4_symlink_inode_operations;
4890			ext4_set_aops(inode);
4891		}
4892		inode_nohighmem(inode);
4893	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4894	      S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
4895		inode->i_op = &ext4_special_inode_operations;
4896		if (raw_inode->i_block[0])
4897			init_special_inode(inode, inode->i_mode,
4898			   old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4899		else
4900			init_special_inode(inode, inode->i_mode,
4901			   new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
4902	} else if (ino == EXT4_BOOT_LOADER_INO) {
4903		make_bad_inode(inode);
4904	} else {
4905		ret = -EFSCORRUPTED;
4906		ext4_error_inode(inode, function, line, 0,
4907				 "iget: bogus i_mode (%o)", inode->i_mode);
4908		goto bad_inode;
4909	}
4910	if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb))
4911		ext4_error_inode(inode, function, line, 0,
4912				 "casefold flag without casefold feature");
4913	brelse(iloc.bh);
4914
4915	unlock_new_inode(inode);
4916	return inode;
4917
4918bad_inode:
4919	brelse(iloc.bh);
4920	iget_failed(inode);
4921	return ERR_PTR(ret);
4922}
4923
4924static int ext4_inode_blocks_set(handle_t *handle,
4925				struct ext4_inode *raw_inode,
4926				struct ext4_inode_info *ei)
4927{
4928	struct inode *inode = &(ei->vfs_inode);
4929	u64 i_blocks = READ_ONCE(inode->i_blocks);
4930	struct super_block *sb = inode->i_sb;
4931
4932	if (i_blocks <= ~0U) {
4933		/*
4934		 * i_blocks can be represented in a 32 bit variable
4935		 * as multiple of 512 bytes
4936		 */
4937		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4938		raw_inode->i_blocks_high = 0;
4939		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4940		return 0;
4941	}
4942	if (!ext4_has_feature_huge_file(sb))
4943		return -EFBIG;
4944
4945	if (i_blocks <= 0xffffffffffffULL) {
4946		/*
4947		 * i_blocks can be represented in a 48 bit variable
4948		 * as multiple of 512 bytes
4949		 */
4950		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4951		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4952		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4953	} else {
4954		ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4955		/* i_block is stored in file system block size */
4956		i_blocks = i_blocks >> (inode->i_blkbits - 9);
4957		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4958		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4959	}
4960	return 0;
4961}
4962
4963static void __ext4_update_other_inode_time(struct super_block *sb,
4964					   unsigned long orig_ino,
4965					   unsigned long ino,
4966					   struct ext4_inode *raw_inode)
4967{
4968	struct inode *inode;
4969
4970	inode = find_inode_by_ino_rcu(sb, ino);
4971	if (!inode)
4972		return;
4973
4974	if (!inode_is_dirtytime_only(inode))
4975		return;
4976
4977	spin_lock(&inode->i_lock);
4978	if (inode_is_dirtytime_only(inode)) {
4979		struct ext4_inode_info	*ei = EXT4_I(inode);
4980
4981		inode->i_state &= ~I_DIRTY_TIME;
4982		spin_unlock(&inode->i_lock);
4983
4984		spin_lock(&ei->i_raw_lock);
4985		EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
4986		EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
4987		EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
4988		ext4_inode_csum_set(inode, raw_inode, ei);
4989		spin_unlock(&ei->i_raw_lock);
4990		trace_ext4_other_inode_update_time(inode, orig_ino);
4991		return;
4992	}
4993	spin_unlock(&inode->i_lock);
4994}
4995
4996/*
4997 * Opportunistically update the other time fields for other inodes in
4998 * the same inode table block.
4999 */
5000static void ext4_update_other_inodes_time(struct super_block *sb,
5001					  unsigned long orig_ino, char *buf)
5002{
5003	unsigned long ino;
5004	int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5005	int inode_size = EXT4_INODE_SIZE(sb);
5006
5007	/*
5008	 * Calculate the first inode in the inode table block.  Inode
5009	 * numbers are one-based.  That is, the first inode in a block
5010	 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
5011	 */
5012	ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
5013	rcu_read_lock();
5014	for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
5015		if (ino == orig_ino)
5016			continue;
5017		__ext4_update_other_inode_time(sb, orig_ino, ino,
5018					       (struct ext4_inode *)buf);
5019	}
5020	rcu_read_unlock();
5021}
5022
5023/*
5024 * Post the struct inode info into an on-disk inode location in the
5025 * buffer-cache.  This gobbles the caller's reference to the
5026 * buffer_head in the inode location struct.
5027 *
5028 * The caller must have write access to iloc->bh.
5029 */
5030static int ext4_do_update_inode(handle_t *handle,
5031				struct inode *inode,
5032				struct ext4_iloc *iloc)
5033{
5034	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5035	struct ext4_inode_info *ei = EXT4_I(inode);
5036	struct buffer_head *bh = iloc->bh;
5037	struct super_block *sb = inode->i_sb;
5038	int err = 0, block;
5039	int need_datasync = 0, set_large_file = 0;
5040	uid_t i_uid;
5041	gid_t i_gid;
5042	projid_t i_projid;
5043
5044	spin_lock(&ei->i_raw_lock);
5045
5046	/* For fields not tracked in the in-memory inode,
5047	 * initialise them to zero for new inodes. */
5048	if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
5049		memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5050
5051	err = ext4_inode_blocks_set(handle, raw_inode, ei);
5052	if (err) {
5053		spin_unlock(&ei->i_raw_lock);
5054		goto out_brelse;
5055	}
5056
5057	raw_inode->i_mode = cpu_to_le16(inode->i_mode);
5058	i_uid = i_uid_read(inode);
5059	i_gid = i_gid_read(inode);
5060	i_projid = from_kprojid(&init_user_ns, ei->i_projid);
5061	if (!(test_opt(inode->i_sb, NO_UID32))) {
5062		raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
5063		raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
5064/*
5065 * Fix up interoperability with old kernels. Otherwise, old inodes get
5066 * re-used with the upper 16 bits of the uid/gid intact
5067 */
5068		if (ei->i_dtime && list_empty(&ei->i_orphan)) {
5069			raw_inode->i_uid_high = 0;
5070			raw_inode->i_gid_high = 0;
5071		} else {
5072			raw_inode->i_uid_high =
5073				cpu_to_le16(high_16_bits(i_uid));
5074			raw_inode->i_gid_high =
5075				cpu_to_le16(high_16_bits(i_gid));
5076		}
5077	} else {
5078		raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
5079		raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
5080		raw_inode->i_uid_high = 0;
5081		raw_inode->i_gid_high = 0;
5082	}
5083	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
5084
5085	EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
5086	EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
5087	EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
5088	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
5089
5090	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
5091	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
5092	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
5093		raw_inode->i_file_acl_high =
5094			cpu_to_le16(ei->i_file_acl >> 32);
5095	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
5096	if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) {
5097		ext4_isize_set(raw_inode, ei->i_disksize);
5098		need_datasync = 1;
5099	}
5100	if (ei->i_disksize > 0x7fffffffULL) {
5101		if (!ext4_has_feature_large_file(sb) ||
5102				EXT4_SB(sb)->s_es->s_rev_level ==
5103		    cpu_to_le32(EXT4_GOOD_OLD_REV))
5104			set_large_file = 1;
5105	}
5106	raw_inode->i_generation = cpu_to_le32(inode->i_generation);
5107	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
5108		if (old_valid_dev(inode->i_rdev)) {
5109			raw_inode->i_block[0] =
5110				cpu_to_le32(old_encode_dev(inode->i_rdev));
5111			raw_inode->i_block[1] = 0;
5112		} else {
5113			raw_inode->i_block[0] = 0;
5114			raw_inode->i_block[1] =
5115				cpu_to_le32(new_encode_dev(inode->i_rdev));
5116			raw_inode->i_block[2] = 0;
5117		}
5118	} else if (!ext4_has_inline_data(inode)) {
5119		for (block = 0; block < EXT4_N_BLOCKS; block++)
5120			raw_inode->i_block[block] = ei->i_data[block];
5121	}
5122
5123	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5124		u64 ivers = ext4_inode_peek_iversion(inode);
5125
5126		raw_inode->i_disk_version = cpu_to_le32(ivers);
5127		if (ei->i_extra_isize) {
5128			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5129				raw_inode->i_version_hi =
5130					cpu_to_le32(ivers >> 32);
5131			raw_inode->i_extra_isize =
5132				cpu_to_le16(ei->i_extra_isize);
5133		}
5134	}
5135
5136	BUG_ON(!ext4_has_feature_project(inode->i_sb) &&
5137	       i_projid != EXT4_DEF_PROJID);
5138
5139	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5140	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5141		raw_inode->i_projid = cpu_to_le32(i_projid);
5142
5143	ext4_inode_csum_set(inode, raw_inode, ei);
5144	spin_unlock(&ei->i_raw_lock);
5145	if (inode->i_sb->s_flags & SB_LAZYTIME)
5146		ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5147					      bh->b_data);
5148
5149	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5150	err = ext4_handle_dirty_metadata(handle, NULL, bh);
5151	if (err)
5152		goto out_brelse;
5153	ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5154	if (set_large_file) {
5155		BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5156		err = ext4_journal_get_write_access(handle, sb,
5157						    EXT4_SB(sb)->s_sbh,
5158						    EXT4_JTR_NONE);
5159		if (err)
5160			goto out_brelse;
5161		lock_buffer(EXT4_SB(sb)->s_sbh);
5162		ext4_set_feature_large_file(sb);
5163		ext4_superblock_csum_set(sb);
5164		unlock_buffer(EXT4_SB(sb)->s_sbh);
5165		ext4_handle_sync(handle);
5166		err = ext4_handle_dirty_metadata(handle, NULL,
5167						 EXT4_SB(sb)->s_sbh);
5168	}
5169	ext4_update_inode_fsync_trans(handle, inode, need_datasync);
5170out_brelse:
5171	brelse(bh);
5172	ext4_std_error(inode->i_sb, err);
5173	return err;
5174}
5175
5176/*
5177 * ext4_write_inode()
5178 *
5179 * We are called from a few places:
5180 *
5181 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
5182 *   Here, there will be no transaction running. We wait for any running
5183 *   transaction to commit.
5184 *
5185 * - Within flush work (sys_sync(), kupdate and such).
5186 *   We wait on commit, if told to.
5187 *
5188 * - Within iput_final() -> write_inode_now()
5189 *   We wait on commit, if told to.
5190 *
5191 * In all cases it is actually safe for us to return without doing anything,
5192 * because the inode has been copied into a raw inode buffer in
5193 * ext4_mark_inode_dirty().  This is a correctness thing for WB_SYNC_ALL
5194 * writeback.
5195 *
5196 * Note that we are absolutely dependent upon all inode dirtiers doing the
5197 * right thing: they *must* call mark_inode_dirty() after dirtying info in
5198 * which we are interested.
5199 *
5200 * It would be a bug for them to not do this.  The code:
5201 *
5202 *	mark_inode_dirty(inode)
5203 *	stuff();
5204 *	inode->i_size = expr;
5205 *
5206 * is in error because write_inode() could occur while `stuff()' is running,
5207 * and the new i_size will be lost.  Plus the inode will no longer be on the
5208 * superblock's dirty inode list.
5209 */
5210int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5211{
5212	int err;
5213
5214	if (WARN_ON_ONCE(current->flags & PF_MEMALLOC) ||
5215	    sb_rdonly(inode->i_sb))
5216		return 0;
5217
5218	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5219		return -EIO;
5220
5221	if (EXT4_SB(inode->i_sb)->s_journal) {
5222		if (ext4_journal_current_handle()) {
5223			jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
5224			dump_stack();
5225			return -EIO;
5226		}
5227
5228		/*
5229		 * No need to force transaction in WB_SYNC_NONE mode. Also
5230		 * ext4_sync_fs() will force the commit after everything is
5231		 * written.
5232		 */
5233		if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5234			return 0;
5235
5236		err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
5237						EXT4_I(inode)->i_sync_tid);
5238	} else {
5239		struct ext4_iloc iloc;
5240
5241		err = __ext4_get_inode_loc_noinmem(inode, &iloc);
5242		if (err)
5243			return err;
5244		/*
5245		 * sync(2) will flush the whole buffer cache. No need to do
5246		 * it here separately for each inode.
5247		 */
5248		if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5249			sync_dirty_buffer(iloc.bh);
5250		if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5251			ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5252					       "IO error syncing inode");
5253			err = -EIO;
5254		}
5255		brelse(iloc.bh);
5256	}
5257	return err;
5258}
5259
5260/*
5261 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate
5262 * buffers that are attached to a page stradding i_size and are undergoing
5263 * commit. In that case we have to wait for commit to finish and try again.
5264 */
5265static void ext4_wait_for_tail_page_commit(struct inode *inode)
5266{
5267	struct page *page;
5268	unsigned offset;
5269	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5270	tid_t commit_tid = 0;
5271	int ret;
5272
5273	offset = inode->i_size & (PAGE_SIZE - 1);
5274	/*
5275	 * If the page is fully truncated, we don't need to wait for any commit
5276	 * (and we even should not as __ext4_journalled_invalidatepage() may
5277	 * strip all buffers from the page but keep the page dirty which can then
5278	 * confuse e.g. concurrent ext4_writepage() seeing dirty page without
5279	 * buffers). Also we don't need to wait for any commit if all buffers in
5280	 * the page remain valid. This is most beneficial for the common case of
5281	 * blocksize == PAGESIZE.
5282	 */
5283	if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
5284		return;
5285	while (1) {
5286		page = find_lock_page(inode->i_mapping,
5287				      inode->i_size >> PAGE_SHIFT);
5288		if (!page)
5289			return;
5290		ret = __ext4_journalled_invalidatepage(page, offset,
5291						PAGE_SIZE - offset);
5292		unlock_page(page);
5293		put_page(page);
5294		if (ret != -EBUSY)
5295			return;
5296		commit_tid = 0;
5297		read_lock(&journal->j_state_lock);
5298		if (journal->j_committing_transaction)
5299			commit_tid = journal->j_committing_transaction->t_tid;
5300		read_unlock(&journal->j_state_lock);
5301		if (commit_tid)
5302			jbd2_log_wait_commit(journal, commit_tid);
5303	}
5304}
5305
5306/*
5307 * ext4_setattr()
5308 *
5309 * Called from notify_change.
5310 *
5311 * We want to trap VFS attempts to truncate the file as soon as
5312 * possible.  In particular, we want to make sure that when the VFS
5313 * shrinks i_size, we put the inode on the orphan list and modify
5314 * i_disksize immediately, so that during the subsequent flushing of
5315 * dirty pages and freeing of disk blocks, we can guarantee that any
5316 * commit will leave the blocks being flushed in an unused state on
5317 * disk.  (On recovery, the inode will get truncated and the blocks will
5318 * be freed, so we have a strong guarantee that no future commit will
5319 * leave these blocks visible to the user.)
5320 *
5321 * Another thing we have to assure is that if we are in ordered mode
5322 * and inode is still attached to the committing transaction, we must
5323 * we start writeout of all the dirty pages which are being truncated.
5324 * This way we are sure that all the data written in the previous
5325 * transaction are already on disk (truncate waits for pages under
5326 * writeback).
5327 *
5328 * Called with inode->i_mutex down.
5329 */
5330int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
5331		 struct iattr *attr)
5332{
5333	struct inode *inode = d_inode(dentry);
5334	int error, rc = 0;
5335	int orphan = 0;
5336	const unsigned int ia_valid = attr->ia_valid;
5337
5338	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5339		return -EIO;
5340
5341	if (unlikely(IS_IMMUTABLE(inode)))
5342		return -EPERM;
5343
5344	if (unlikely(IS_APPEND(inode) &&
5345		     (ia_valid & (ATTR_MODE | ATTR_UID |
5346				  ATTR_GID | ATTR_TIMES_SET))))
5347		return -EPERM;
5348
5349	error = setattr_prepare(mnt_userns, dentry, attr);
5350	if (error)
5351		return error;
5352
5353	error = fscrypt_prepare_setattr(dentry, attr);
5354	if (error)
5355		return error;
5356
5357	error = fsverity_prepare_setattr(dentry, attr);
5358	if (error)
5359		return error;
5360
5361	if (is_quota_modification(inode, attr)) {
5362		error = dquot_initialize(inode);
5363		if (error)
5364			return error;
5365	}
5366	ext4_fc_start_update(inode);
5367	if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
5368	    (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
5369		handle_t *handle;
5370
5371		/* (user+group)*(old+new) structure, inode write (sb,
5372		 * inode block, ? - but truncate inode update has it) */
5373		handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5374			(EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5375			 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5376		if (IS_ERR(handle)) {
5377			error = PTR_ERR(handle);
5378			goto err_out;
5379		}
5380
5381		/* dquot_transfer() calls back ext4_get_inode_usage() which
5382		 * counts xattr inode references.
5383		 */
5384		down_read(&EXT4_I(inode)->xattr_sem);
5385		error = dquot_transfer(inode, attr);
5386		up_read(&EXT4_I(inode)->xattr_sem);
5387
5388		if (error) {
5389			ext4_journal_stop(handle);
5390			ext4_fc_stop_update(inode);
5391			return error;
5392		}
5393		/* Update corresponding info in inode so that everything is in
5394		 * one transaction */
5395		if (attr->ia_valid & ATTR_UID)
5396			inode->i_uid = attr->ia_uid;
5397		if (attr->ia_valid & ATTR_GID)
5398			inode->i_gid = attr->ia_gid;
5399		error = ext4_mark_inode_dirty(handle, inode);
5400		ext4_journal_stop(handle);
5401		if (unlikely(error)) {
5402			ext4_fc_stop_update(inode);
5403			return error;
5404		}
5405	}
5406
5407	if (attr->ia_valid & ATTR_SIZE) {
5408		handle_t *handle;
5409		loff_t oldsize = inode->i_size;
5410		int shrink = (attr->ia_size < inode->i_size);
5411
5412		if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
5413			struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5414
5415			if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5416				ext4_fc_stop_update(inode);
5417				return -EFBIG;
5418			}
5419		}
5420		if (!S_ISREG(inode->i_mode)) {
5421			ext4_fc_stop_update(inode);
5422			return -EINVAL;
5423		}
5424
5425		if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
5426			inode_inc_iversion(inode);
5427
5428		if (shrink) {
5429			if (ext4_should_order_data(inode)) {
5430				error = ext4_begin_ordered_truncate(inode,
5431							    attr->ia_size);
5432				if (error)
5433					goto err_out;
5434			}
5435			/*
5436			 * Blocks are going to be removed from the inode. Wait
5437			 * for dio in flight.
5438			 */
5439			inode_dio_wait(inode);
5440		}
5441
5442		down_write(&EXT4_I(inode)->i_mmap_sem);
5443
5444		rc = ext4_break_layouts(inode);
5445		if (rc) {
5446			up_write(&EXT4_I(inode)->i_mmap_sem);
5447			goto err_out;
5448		}
5449
5450		if (attr->ia_size != inode->i_size) {
5451			handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5452			if (IS_ERR(handle)) {
5453				error = PTR_ERR(handle);
5454				goto out_mmap_sem;
5455			}
5456			if (ext4_handle_valid(handle) && shrink) {
5457				error = ext4_orphan_add(handle, inode);
5458				orphan = 1;
5459			}
5460			/*
5461			 * Update c/mtime on truncate up, ext4_truncate() will
5462			 * update c/mtime in shrink case below
5463			 */
5464			if (!shrink) {
5465				inode->i_mtime = current_time(inode);
5466				inode->i_ctime = inode->i_mtime;
5467			}
5468
5469			if (shrink)
5470				ext4_fc_track_range(handle, inode,
5471					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5472					inode->i_sb->s_blocksize_bits,
5473					(oldsize > 0 ? oldsize - 1 : 0) >>
5474					inode->i_sb->s_blocksize_bits);
5475			else
5476				ext4_fc_track_range(
5477					handle, inode,
5478					(oldsize > 0 ? oldsize - 1 : oldsize) >>
5479					inode->i_sb->s_blocksize_bits,
5480					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5481					inode->i_sb->s_blocksize_bits);
5482
5483			down_write(&EXT4_I(inode)->i_data_sem);
5484			EXT4_I(inode)->i_disksize = attr->ia_size;
5485			rc = ext4_mark_inode_dirty(handle, inode);
5486			if (!error)
5487				error = rc;
5488			/*
5489			 * We have to update i_size under i_data_sem together
5490			 * with i_disksize to avoid races with writeback code
5491			 * running ext4_wb_update_i_disksize().
5492			 */
5493			if (!error)
5494				i_size_write(inode, attr->ia_size);
5495			up_write(&EXT4_I(inode)->i_data_sem);
5496			ext4_journal_stop(handle);
5497			if (error)
5498				goto out_mmap_sem;
5499			if (!shrink) {
5500				pagecache_isize_extended(inode, oldsize,
5501							 inode->i_size);
5502			} else if (ext4_should_journal_data(inode)) {
5503				ext4_wait_for_tail_page_commit(inode);
5504			}
5505		}
5506
5507		/*
5508		 * Truncate pagecache after we've waited for commit
5509		 * in data=journal mode to make pages freeable.
5510		 */
5511		truncate_pagecache(inode, inode->i_size);
5512		/*
5513		 * Call ext4_truncate() even if i_size didn't change to
5514		 * truncate possible preallocated blocks.
5515		 */
5516		if (attr->ia_size <= oldsize) {
5517			rc = ext4_truncate(inode);
5518			if (rc)
5519				error = rc;
5520		}
5521out_mmap_sem:
5522		up_write(&EXT4_I(inode)->i_mmap_sem);
5523	}
5524
5525	if (!error) {
5526		setattr_copy(mnt_userns, inode, attr);
5527		mark_inode_dirty(inode);
5528	}
5529
5530	/*
5531	 * If the call to ext4_truncate failed to get a transaction handle at
5532	 * all, we need to clean up the in-core orphan list manually.
5533	 */
5534	if (orphan && inode->i_nlink)
5535		ext4_orphan_del(NULL, inode);
5536
5537	if (!error && (ia_valid & ATTR_MODE))
5538		rc = posix_acl_chmod(mnt_userns, inode, inode->i_mode);
5539
5540err_out:
5541	if  (error)
5542		ext4_std_error(inode->i_sb, error);
5543	if (!error)
5544		error = rc;
5545	ext4_fc_stop_update(inode);
5546	return error;
5547}
5548
5549int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path,
5550		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
5551{
5552	struct inode *inode = d_inode(path->dentry);
5553	struct ext4_inode *raw_inode;
5554	struct ext4_inode_info *ei = EXT4_I(inode);
5555	unsigned int flags;
5556
5557	if ((request_mask & STATX_BTIME) &&
5558	    EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
5559		stat->result_mask |= STATX_BTIME;
5560		stat->btime.tv_sec = ei->i_crtime.tv_sec;
5561		stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5562	}
5563
5564	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5565	if (flags & EXT4_APPEND_FL)
5566		stat->attributes |= STATX_ATTR_APPEND;
5567	if (flags & EXT4_COMPR_FL)
5568		stat->attributes |= STATX_ATTR_COMPRESSED;
5569	if (flags & EXT4_ENCRYPT_FL)
5570		stat->attributes |= STATX_ATTR_ENCRYPTED;
5571	if (flags & EXT4_IMMUTABLE_FL)
5572		stat->attributes |= STATX_ATTR_IMMUTABLE;
5573	if (flags & EXT4_NODUMP_FL)
5574		stat->attributes |= STATX_ATTR_NODUMP;
5575	if (flags & EXT4_VERITY_FL)
5576		stat->attributes |= STATX_ATTR_VERITY;
5577
5578	stat->attributes_mask |= (STATX_ATTR_APPEND |
5579				  STATX_ATTR_COMPRESSED |
5580				  STATX_ATTR_ENCRYPTED |
5581				  STATX_ATTR_IMMUTABLE |
5582				  STATX_ATTR_NODUMP |
5583				  STATX_ATTR_VERITY);
5584
5585	generic_fillattr(mnt_userns, inode, stat);
5586	return 0;
5587}
5588
5589int ext4_file_getattr(struct user_namespace *mnt_userns,
5590		      const struct path *path, struct kstat *stat,
5591		      u32 request_mask, unsigned int query_flags)
5592{
5593	struct inode *inode = d_inode(path->dentry);
5594	u64 delalloc_blocks;
5595
5596	ext4_getattr(mnt_userns, path, stat, request_mask, query_flags);
5597
5598	/*
5599	 * If there is inline data in the inode, the inode will normally not
5600	 * have data blocks allocated (it may have an external xattr block).
5601	 * Report at least one sector for such files, so tools like tar, rsync,
5602	 * others don't incorrectly think the file is completely sparse.
5603	 */
5604	if (unlikely(ext4_has_inline_data(inode)))
5605		stat->blocks += (stat->size + 511) >> 9;
5606
5607	/*
5608	 * We can't update i_blocks if the block allocation is delayed
5609	 * otherwise in the case of system crash before the real block
5610	 * allocation is done, we will have i_blocks inconsistent with
5611	 * on-disk file blocks.
5612	 * We always keep i_blocks updated together with real
5613	 * allocation. But to not confuse with user, stat
5614	 * will return the blocks that include the delayed allocation
5615	 * blocks for this file.
5616	 */
5617	delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
5618				   EXT4_I(inode)->i_reserved_data_blocks);
5619	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
5620	return 0;
5621}
5622
5623static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5624				   int pextents)
5625{
5626	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
5627		return ext4_ind_trans_blocks(inode, lblocks);
5628	return ext4_ext_index_trans_blocks(inode, pextents);
5629}
5630
5631/*
5632 * Account for index blocks, block groups bitmaps and block group
5633 * descriptor blocks if modify datablocks and index blocks
5634 * worse case, the indexs blocks spread over different block groups
5635 *
5636 * If datablocks are discontiguous, they are possible to spread over
5637 * different block groups too. If they are contiguous, with flexbg,
5638 * they could still across block group boundary.
5639 *
5640 * Also account for superblock, inode, quota and xattr blocks
5641 */
5642static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5643				  int pextents)
5644{
5645	ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5646	int gdpblocks;
5647	int idxblocks;
5648	int ret = 0;
5649
5650	/*
5651	 * How many index blocks need to touch to map @lblocks logical blocks
5652	 * to @pextents physical extents?
5653	 */
5654	idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
5655
5656	ret = idxblocks;
5657
5658	/*
5659	 * Now let's see how many group bitmaps and group descriptors need
5660	 * to account
5661	 */
5662	groups = idxblocks + pextents;
5663	gdpblocks = groups;
5664	if (groups > ngroups)
5665		groups = ngroups;
5666	if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5667		gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5668
5669	/* bitmaps and block group descriptor blocks */
5670	ret += groups + gdpblocks;
5671
5672	/* Blocks for super block, inode, quota and xattr blocks */
5673	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
5674
5675	return ret;
5676}
5677
5678/*
5679 * Calculate the total number of credits to reserve to fit
5680 * the modification of a single pages into a single transaction,
5681 * which may include multiple chunks of block allocations.
5682 *
5683 * This could be called via ext4_write_begin()
5684 *
5685 * We need to consider the worse case, when
5686 * one new block per extent.
5687 */
5688int ext4_writepage_trans_blocks(struct inode *inode)
5689{
5690	int bpp = ext4_journal_blocks_per_page(inode);
5691	int ret;
5692
5693	ret = ext4_meta_trans_blocks(inode, bpp, bpp);
5694
5695	/* Account for data blocks for journalled mode */
5696	if (ext4_should_journal_data(inode))
5697		ret += bpp;
5698	return ret;
5699}
5700
5701/*
5702 * Calculate the journal credits for a chunk of data modification.
5703 *
5704 * This is called from DIO, fallocate or whoever calling
5705 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
5706 *
5707 * journal buffers for data blocks are not included here, as DIO
5708 * and fallocate do no need to journal data buffers.
5709 */
5710int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5711{
5712	return ext4_meta_trans_blocks(inode, nrblocks, 1);
5713}
5714
5715/*
5716 * The caller must have previously called ext4_reserve_inode_write().
5717 * Give this, we know that the caller already has write access to iloc->bh.
5718 */
5719int ext4_mark_iloc_dirty(handle_t *handle,
5720			 struct inode *inode, struct ext4_iloc *iloc)
5721{
5722	int err = 0;
5723
5724	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
5725		put_bh(iloc->bh);
5726		return -EIO;
5727	}
5728	ext4_fc_track_inode(handle, inode);
5729
5730	if (IS_I_VERSION(inode))
5731		inode_inc_iversion(inode);
5732
5733	/* the do_update_inode consumes one bh->b_count */
5734	get_bh(iloc->bh);
5735
5736	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
5737	err = ext4_do_update_inode(handle, inode, iloc);
5738	put_bh(iloc->bh);
5739	return err;
5740}
5741
5742/*
5743 * On success, We end up with an outstanding reference count against
5744 * iloc->bh.  This _must_ be cleaned up later.
5745 */
5746
5747int
5748ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5749			 struct ext4_iloc *iloc)
5750{
5751	int err;
5752
5753	if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5754		return -EIO;
5755
5756	err = ext4_get_inode_loc(inode, iloc);
5757	if (!err) {
5758		BUFFER_TRACE(iloc->bh, "get_write_access");
5759		err = ext4_journal_get_write_access(handle, inode->i_sb,
5760						    iloc->bh, EXT4_JTR_NONE);
5761		if (err) {
5762			brelse(iloc->bh);
5763			iloc->bh = NULL;
5764		}
5765	}
5766	ext4_std_error(inode->i_sb, err);
5767	return err;
5768}
5769
5770static int __ext4_expand_extra_isize(struct inode *inode,
5771				     unsigned int new_extra_isize,
5772				     struct ext4_iloc *iloc,
5773				     handle_t *handle, int *no_expand)
5774{
5775	struct ext4_inode *raw_inode;
5776	struct ext4_xattr_ibody_header *header;
5777	unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
5778	struct ext4_inode_info *ei = EXT4_I(inode);
5779	int error;
5780
5781	/* this was checked at iget time, but double check for good measure */
5782	if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
5783	    (ei->i_extra_isize & 3)) {
5784		EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
5785				 ei->i_extra_isize,
5786				 EXT4_INODE_SIZE(inode->i_sb));
5787		return -EFSCORRUPTED;
5788	}
5789	if ((new_extra_isize < ei->i_extra_isize) ||
5790	    (new_extra_isize < 4) ||
5791	    (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
5792		return -EINVAL;	/* Should never happen */
5793
5794	raw_inode = ext4_raw_inode(iloc);
5795
5796	header = IHDR(inode, raw_inode);
5797
5798	/* No extended attributes present */
5799	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5800	    header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5801		memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5802		       EXT4_I(inode)->i_extra_isize, 0,
5803		       new_extra_isize - EXT4_I(inode)->i_extra_isize);
5804		EXT4_I(inode)->i_extra_isize = new_extra_isize;
5805		return 0;
5806	}
5807
5808	/* try to expand with EAs present */
5809	error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5810					   raw_inode, handle);
5811	if (error) {
5812		/*
5813		 * Inode size expansion failed; don't try again
5814		 */
5815		*no_expand = 1;
5816	}
5817
5818	return error;
5819}
5820
5821/*
5822 * Expand an inode by new_extra_isize bytes.
5823 * Returns 0 on success or negative error number on failure.
5824 */
5825static int ext4_try_to_expand_extra_isize(struct inode *inode,
5826					  unsigned int new_extra_isize,
5827					  struct ext4_iloc iloc,
5828					  handle_t *handle)
5829{
5830	int no_expand;
5831	int error;
5832
5833	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
5834		return -EOVERFLOW;
5835
5836	/*
5837	 * In nojournal mode, we can immediately attempt to expand
5838	 * the inode.  When journaled, we first need to obtain extra
5839	 * buffer credits since we may write into the EA block
5840	 * with this same handle. If journal_extend fails, then it will
5841	 * only result in a minor loss of functionality for that inode.
5842	 * If this is felt to be critical, then e2fsck should be run to
5843	 * force a large enough s_min_extra_isize.
5844	 */
5845	if (ext4_journal_extend(handle,
5846				EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
5847		return -ENOSPC;
5848
5849	if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
5850		return -EBUSY;
5851
5852	error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
5853					  handle, &no_expand);
5854	ext4_write_unlock_xattr(inode, &no_expand);
5855
5856	return error;
5857}
5858
5859int ext4_expand_extra_isize(struct inode *inode,
5860			    unsigned int new_extra_isize,
5861			    struct ext4_iloc *iloc)
5862{
5863	handle_t *handle;
5864	int no_expand;
5865	int error, rc;
5866
5867	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5868		brelse(iloc->bh);
5869		return -EOVERFLOW;
5870	}
5871
5872	handle = ext4_journal_start(inode, EXT4_HT_INODE,
5873				    EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5874	if (IS_ERR(handle)) {
5875		error = PTR_ERR(handle);
5876		brelse(iloc->bh);
5877		return error;
5878	}
5879
5880	ext4_write_lock_xattr(inode, &no_expand);
5881
5882	BUFFER_TRACE(iloc->bh, "get_write_access");
5883	error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
5884					      EXT4_JTR_NONE);
5885	if (error) {
5886		brelse(iloc->bh);
5887		goto out_unlock;
5888	}
5889
5890	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
5891					  handle, &no_expand);
5892
5893	rc = ext4_mark_iloc_dirty(handle, inode, iloc);
5894	if (!error)
5895		error = rc;
5896
5897out_unlock:
5898	ext4_write_unlock_xattr(inode, &no_expand);
5899	ext4_journal_stop(handle);
5900	return error;
5901}
5902
5903/*
5904 * What we do here is to mark the in-core inode as clean with respect to inode
5905 * dirtiness (it may still be data-dirty).
5906 * This means that the in-core inode may be reaped by prune_icache
5907 * without having to perform any I/O.  This is a very good thing,
5908 * because *any* task may call prune_icache - even ones which
5909 * have a transaction open against a different journal.
5910 *
5911 * Is this cheating?  Not really.  Sure, we haven't written the
5912 * inode out, but prune_icache isn't a user-visible syncing function.
5913 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
5914 * we start and wait on commits.
5915 */
5916int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
5917				const char *func, unsigned int line)
5918{
5919	struct ext4_iloc iloc;
5920	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5921	int err;
5922
5923	might_sleep();
5924	trace_ext4_mark_inode_dirty(inode, _RET_IP_);
5925	err = ext4_reserve_inode_write(handle, inode, &iloc);
5926	if (err)
5927		goto out;
5928
5929	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
5930		ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
5931					       iloc, handle);
5932
5933	err = ext4_mark_iloc_dirty(handle, inode, &iloc);
5934out:
5935	if (unlikely(err))
5936		ext4_error_inode_err(inode, func, line, 0, err,
5937					"mark_inode_dirty error");
5938	return err;
5939}
5940
5941/*
5942 * ext4_dirty_inode() is called from __mark_inode_dirty()
5943 *
5944 * We're really interested in the case where a file is being extended.
5945 * i_size has been changed by generic_commit_write() and we thus need
5946 * to include the updated inode in the current transaction.
5947 *
5948 * Also, dquot_alloc_block() will always dirty the inode when blocks
5949 * are allocated to the file.
5950 *
5951 * If the inode is marked synchronous, we don't honour that here - doing
5952 * so would cause a commit on atime updates, which we don't bother doing.
5953 * We handle synchronous inodes at the highest possible level.
5954 */
5955void ext4_dirty_inode(struct inode *inode, int flags)
5956{
5957	handle_t *handle;
5958
5959	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
5960	if (IS_ERR(handle))
5961		return;
5962	ext4_mark_inode_dirty(handle, inode);
5963	ext4_journal_stop(handle);
5964}
5965
5966int ext4_change_inode_journal_flag(struct inode *inode, int val)
5967{
5968	journal_t *journal;
5969	handle_t *handle;
5970	int err;
5971	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5972
5973	/*
5974	 * We have to be very careful here: changing a data block's
5975	 * journaling status dynamically is dangerous.  If we write a
5976	 * data block to the journal, change the status and then delete
5977	 * that block, we risk forgetting to revoke the old log record
5978	 * from the journal and so a subsequent replay can corrupt data.
5979	 * So, first we make sure that the journal is empty and that
5980	 * nobody is changing anything.
5981	 */
5982
5983	journal = EXT4_JOURNAL(inode);
5984	if (!journal)
5985		return 0;
5986	if (is_journal_aborted(journal))
5987		return -EROFS;
5988
5989	/* Wait for all existing dio workers */
5990	inode_dio_wait(inode);
5991
5992	/*
5993	 * Before flushing the journal and switching inode's aops, we have
5994	 * to flush all dirty data the inode has. There can be outstanding
5995	 * delayed allocations, there can be unwritten extents created by
5996	 * fallocate or buffered writes in dioread_nolock mode covered by
5997	 * dirty data which can be converted only after flushing the dirty
5998	 * data (and journalled aops don't know how to handle these cases).
5999	 */
6000	if (val) {
6001		down_write(&EXT4_I(inode)->i_mmap_sem);
6002		err = filemap_write_and_wait(inode->i_mapping);
6003		if (err < 0) {
6004			up_write(&EXT4_I(