fs/ext4/inode.c

1544 static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1550 	struct inode *inode = mpd->inode;
1554 	if (mpd->first_page >= mpd->next_page)
1557 	mpd->scanned_until_end = 0;
1558 	index = mpd->first_page;
1559 	end   = mpd->next_page - 1;
1582 			if (folio->index < mpd->first_page)
1837 static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)
1839 	mpd->first_page += folio_nr_pages(folio);
1843 static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)
1849 	BUG_ON(folio->index != mpd->first_page);
1864 	size = i_size_read(mpd->inode);
1867 	    !ext4_verity_in_progress(mpd->inode))
1869 	err = ext4_bio_write_folio(&mpd->io_submit, folio, len);
1871 		mpd->wbc->nr_to_write--;
1888  * @mpd - extent of blocks
1899 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
1902 	struct ext4_map_blocks *map = &mpd->map;
1916 		if (!mpd->do_map)
1940  * @mpd - extent of blocks for mapping
1953 static int mpage_process_page_bufs(struct mpage_da_data *mpd,
1958 	struct inode *inode = mpd->inode;
1969 		if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
1971 			if (mpd->map.m_len)
1974 			if (!mpd->do_map)
1981 	if (mpd->map.m_len == 0) {
1982 		err = mpage_submit_folio(mpd, head->b_folio);
1985 		mpage_folio_done(mpd, head->b_folio);
1988 		mpd->scanned_until_end = 1;
1997  * @mpd: description of extent to map, on return next extent to map
2007  * If the given folio is not fully mapped, we update @mpd to the next extent in
2010 static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,
2015 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2019 	int blkbits = mpd->inode->i_blkbits;
2025 		if (lblk < mpd->map.m_lblk)
2027 		if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2032 			mpd->map.m_len = 0;
2033 			mpd->map.m_flags = 0;
2036 			err = mpage_process_page_bufs(mpd, head, bh, lblk);
2039 			if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2045 				io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2070  * @mpd - description of extent to map, on return next extent to map
2080 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2084 	struct inode *inode = mpd->inode;
2092 	start = mpd->map.m_lblk >> bpp_bits;
2093 	end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2095 	pblock = mpd->map.m_pblk;
2105 			err = mpage_process_folio(mpd, folio, &lblk, &pblock,
2115 			err = mpage_submit_folio(mpd, folio);
2118 			mpage_folio_done(mpd, folio);
2123 	mpd->map.m_len = 0;
2124 	mpd->map.m_flags = 0;
2131 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2133 	struct inode *inode = mpd->inode;
2134 	struct ext4_map_blocks *map = &mpd->map;
2167 		if (!mpd->io_submit.io_end->handle &&
2169 			mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2172 		ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2180  * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2181  *				 mpd->len and submit pages underlying it for IO
2184  * @mpd - extent to map
2189  * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2200 				       struct mpage_da_data *mpd,
2203 	struct inode *inode = mpd->inode;
2204 	struct ext4_map_blocks *map = &mpd->map;
2208 	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2216 		err = mpage_map_one_extent(handle, mpd);
2254 		err = mpage_map_and_submit_buffers(mpd);
2264 	disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2325 				      struct mpage_da_data *mpd,
2328 	struct inode *inode = mpd->inode;
2333 	mpd->wbc->nr_to_write--;
2346  * @mpd - where to look for pages
2354  * extent found is returned in @mpd structure (starting at mpd->lblk with
2355  * length mpd->len blocks).
2362 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2364 	struct address_space *mapping = mpd->inode->i_mapping;
2367 	pgoff_t index = mpd->first_page;
2368 	pgoff_t end = mpd->last_page;
2371 	int blkbits = mpd->inode->i_blkbits;
2375 	int bpp = ext4_journal_blocks_per_page(mpd->inode);
2377 	if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2382 	mpd->map.m_len = 0;
2383 	mpd->next_page = index;
2384 	if (ext4_should_journal_data(mpd->inode)) {
2385 		handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,
2408 			if (mpd->wbc->sync_mode == WB_SYNC_NONE &&
2409 			    mpd->wbc->nr_to_write <=
2410 			    mpd->map.m_len >> (PAGE_SHIFT - blkbits))
2414 			if (mpd->map.m_len > 0 && mpd->next_page != folio->index)
2434 			     (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2453 				ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);
2459 			if (mpd->map.m_len == 0)
2460 				mpd->first_page = folio->index;
2461 			mpd->next_page = folio_next_index(folio);
2472 			if (!mpd->can_map) {
2473 				err = mpage_submit_folio(mpd, folio);
2479 						mpd, folio);
2482 					mpd->journalled_more_data = 1;
2484 				mpage_folio_done(mpd, folio);
2486 				/* Add all dirty buffers to mpd */
2490 				err = mpage_process_page_bufs(mpd, head, head,
2500 	mpd->scanned_until_end = 1;
2511 static int ext4_do_writepages(struct mpage_da_data *mpd)
2513 	struct writeback_control *wbc = mpd->wbc;
2519 	struct inode *inode = mpd->inode;
2582 		mpd->can_map = 0;
2587 	mpd->journalled_more_data = 0;
2605 		mpd->first_page = writeback_index;
2606 		mpd->last_page = -1;
2608 		mpd->first_page = wbc->range_start >> PAGE_SHIFT;
2609 		mpd->last_page = wbc->range_end >> PAGE_SHIFT;
2612 	ext4_io_submit_init(&mpd->io_submit, wbc);
2615 		tag_pages_for_writeback(mapping, mpd->first_page,
2616 					mpd->last_page);
2625 	mpd->do_map = 0;
2626 	mpd->scanned_until_end = 0;
2627 	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2628 	if (!mpd->io_submit.io_end) {
2632 	ret = mpage_prepare_extent_to_map(mpd);
2634 	mpage_release_unused_pages(mpd, false);
2636 	ext4_io_submit(&mpd->io_submit);
2637 	ext4_put_io_end_defer(mpd->io_submit.io_end);
2638 	mpd->io_submit.io_end = NULL;
2642 	while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {
2644 		mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2645 		if (!mpd->io_submit.io_end) {
2650 		WARN_ON_ONCE(!mpd->can_map);
2670 			ext4_put_io_end(mpd->io_submit.io_end);
2671 			mpd->io_submit.io_end = NULL;
2674 		mpd->do_map = 1;
2676 		trace_ext4_da_write_pages(inode, mpd->first_page, wbc);
2677 		ret = mpage_prepare_extent_to_map(mpd);
2678 		if (!ret && mpd->map.m_len)
2679 			ret = mpage_map_and_submit_extent(handle, mpd,
2694 			mpd->do_map = 0;
2697 		mpage_release_unused_pages(mpd, give_up_on_write);
2699 		ext4_io_submit(&mpd->io_submit);
2709 			ext4_put_io_end_defer(mpd->io_submit.io_end);
2712 			ext4_put_io_end(mpd->io_submit.io_end);
2713 		mpd->io_submit.io_end = NULL;
2733 		mpd->last_page = writeback_index - 1;
2734 		mpd->first_page = 0;
2744 		mapping->writeback_index = mpd->first_page;
2756 	struct mpage_da_data mpd = {
2768 	ret = ext4_do_writepages(&mpd);
2774 	if (!ret && mpd.journalled_more_data)
2775 		ret = ext4_do_writepages(&mpd);
2789 	struct mpage_da_data mpd = {
2794 	return ext4_do_writepages(&mpd);