data.c revision eadcd6b5
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2017-2018 HUAWEI, Inc.
4 *             https://www.huawei.com/
5 */
6#include "internal.h"
7#include <linux/prefetch.h>
8#include <linux/dax.h>
9#include <trace/events/erofs.h>
10
11struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr)
12{
13	struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;
14	struct page *page;
15
16	page = read_cache_page_gfp(mapping, blkaddr,
17				   mapping_gfp_constraint(mapping, ~__GFP_FS));
18	/* should already be PageUptodate */
19	if (!IS_ERR(page))
20		lock_page(page);
21	return page;
22}
23
24static int erofs_map_blocks_flatmode(struct inode *inode,
25				     struct erofs_map_blocks *map,
26				     int flags)
27{
28	int err = 0;
29	erofs_blk_t nblocks, lastblk;
30	u64 offset = map->m_la;
31	struct erofs_inode *vi = EROFS_I(inode);
32	bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);
33
34	trace_erofs_map_blocks_flatmode_enter(inode, map, flags);
35
36	nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
37	lastblk = nblocks - tailendpacking;
38
39	if (offset >= inode->i_size) {
40		/* leave out-of-bound access unmapped */
41		map->m_flags = 0;
42		map->m_plen = 0;
43		goto out;
44	}
45
46	/* there is no hole in flatmode */
47	map->m_flags = EROFS_MAP_MAPPED;
48
49	if (offset < blknr_to_addr(lastblk)) {
50		map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
51		map->m_plen = blknr_to_addr(lastblk) - offset;
52	} else if (tailendpacking) {
53		/* 2 - inode inline B: inode, [xattrs], inline last blk... */
54		struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
55
56		map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
57			vi->xattr_isize + erofs_blkoff(map->m_la);
58		map->m_plen = inode->i_size - offset;
59
60		/* inline data should be located in one meta block */
61		if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {
62			erofs_err(inode->i_sb,
63				  "inline data cross block boundary @ nid %llu",
64				  vi->nid);
65			DBG_BUGON(1);
66			err = -EFSCORRUPTED;
67			goto err_out;
68		}
69
70		map->m_flags |= EROFS_MAP_META;
71	} else {
72		erofs_err(inode->i_sb,
73			  "internal error @ nid: %llu (size %llu), m_la 0x%llx",
74			  vi->nid, inode->i_size, map->m_la);
75		DBG_BUGON(1);
76		err = -EIO;
77		goto err_out;
78	}
79
80out:
81	map->m_llen = map->m_plen;
82
83err_out:
84	trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0);
85	return err;
86}
87
88static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
89		unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
90{
91	int ret;
92	struct erofs_map_blocks map;
93
94	map.m_la = offset;
95	map.m_llen = length;
96
97	ret = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
98	if (ret < 0)
99		return ret;
100
101	iomap->bdev = inode->i_sb->s_bdev;
102	iomap->dax_dev = EROFS_I_SB(inode)->dax_dev;
103	iomap->offset = map.m_la;
104	iomap->length = map.m_llen;
105	iomap->flags = 0;
106	iomap->private = NULL;
107
108	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
109		iomap->type = IOMAP_HOLE;
110		iomap->addr = IOMAP_NULL_ADDR;
111		if (!iomap->length)
112			iomap->length = length;
113		return 0;
114	}
115
116	if (map.m_flags & EROFS_MAP_META) {
117		struct page *ipage;
118
119		iomap->type = IOMAP_INLINE;
120		ipage = erofs_get_meta_page(inode->i_sb,
121					    erofs_blknr(map.m_pa));
122		if (IS_ERR(ipage))
123			return PTR_ERR(ipage);
124		iomap->inline_data = page_address(ipage) +
125					erofs_blkoff(map.m_pa);
126		iomap->private = ipage;
127	} else {
128		iomap->type = IOMAP_MAPPED;
129		iomap->addr = map.m_pa;
130	}
131	return 0;
132}
133
134static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length,
135		ssize_t written, unsigned int flags, struct iomap *iomap)
136{
137	struct page *ipage = iomap->private;
138
139	if (ipage) {
140		DBG_BUGON(iomap->type != IOMAP_INLINE);
141		unlock_page(ipage);
142		put_page(ipage);
143	} else {
144		DBG_BUGON(iomap->type == IOMAP_INLINE);
145	}
146	return written;
147}
148
149static const struct iomap_ops erofs_iomap_ops = {
150	.iomap_begin = erofs_iomap_begin,
151	.iomap_end = erofs_iomap_end,
152};
153
154int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
155		 u64 start, u64 len)
156{
157	if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
158#ifdef CONFIG_EROFS_FS_ZIP
159		return iomap_fiemap(inode, fieinfo, start, len,
160				    &z_erofs_iomap_report_ops);
161#else
162		return -EOPNOTSUPP;
163#endif
164	}
165	return iomap_fiemap(inode, fieinfo, start, len, &erofs_iomap_ops);
166}
167
168/*
169 * since we dont have write or truncate flows, so no inode
170 * locking needs to be held at the moment.
171 */
172static int erofs_readpage(struct file *file, struct page *page)
173{
174	return iomap_readpage(page, &erofs_iomap_ops);
175}
176
177static void erofs_readahead(struct readahead_control *rac)
178{
179	return iomap_readahead(rac, &erofs_iomap_ops);
180}
181
182static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
183{
184	return iomap_bmap(mapping, block, &erofs_iomap_ops);
185}
186
187static int erofs_prepare_dio(struct kiocb *iocb, struct iov_iter *to)
188{
189	struct inode *inode = file_inode(iocb->ki_filp);
190	loff_t align = iocb->ki_pos | iov_iter_count(to) |
191		iov_iter_alignment(to);
192	struct block_device *bdev = inode->i_sb->s_bdev;
193	unsigned int blksize_mask;
194
195	if (bdev)
196		blksize_mask = (1 << ilog2(bdev_logical_block_size(bdev))) - 1;
197	else
198		blksize_mask = (1 << inode->i_blkbits) - 1;
199
200	if (align & blksize_mask)
201		return -EINVAL;
202	return 0;
203}
204
205static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
206{
207	/* no need taking (shared) inode lock since it's a ro filesystem */
208	if (!iov_iter_count(to))
209		return 0;
210
211#ifdef CONFIG_FS_DAX
212	if (IS_DAX(iocb->ki_filp->f_mapping->host))
213		return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
214#endif
215	if (iocb->ki_flags & IOCB_DIRECT) {
216		int err = erofs_prepare_dio(iocb, to);
217
218		if (!err)
219			return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
220					    NULL, 0);
221		if (err < 0)
222			return err;
223	}
224	return filemap_read(iocb, to, 0);
225}
226
227/* for uncompressed (aligned) files and raw access for other files */
228const struct address_space_operations erofs_raw_access_aops = {
229	.readpage = erofs_readpage,
230	.readahead = erofs_readahead,
231	.bmap = erofs_bmap,
232	.direct_IO = noop_direct_IO,
233};
234
235#ifdef CONFIG_FS_DAX
236static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf,
237		enum page_entry_size pe_size)
238{
239	return dax_iomap_fault(vmf, pe_size, NULL, NULL, &erofs_iomap_ops);
240}
241
242static vm_fault_t erofs_dax_fault(struct vm_fault *vmf)
243{
244	return erofs_dax_huge_fault(vmf, PE_SIZE_PTE);
245}
246
247static const struct vm_operations_struct erofs_dax_vm_ops = {
248	.fault		= erofs_dax_fault,
249	.huge_fault	= erofs_dax_huge_fault,
250};
251
252static int erofs_file_mmap(struct file *file, struct vm_area_struct *vma)
253{
254	if (!IS_DAX(file_inode(file)))
255		return generic_file_readonly_mmap(file, vma);
256
257	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
258		return -EINVAL;
259
260	vma->vm_ops = &erofs_dax_vm_ops;
261	vma->vm_flags |= VM_HUGEPAGE;
262	return 0;
263}
264#else
265#define erofs_file_mmap	generic_file_readonly_mmap
266#endif
267
268const struct file_operations erofs_file_fops = {
269	.llseek		= generic_file_llseek,
270	.read_iter	= erofs_file_read_iter,
271	.mmap		= erofs_file_mmap,
272	.splice_read	= generic_file_splice_read,
273};
274