fs/iomap/buffered-io.c

47 		struct iomap_folio_state *ifs)
51 	return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));
54 static inline bool ifs_block_is_uptodate(struct iomap_folio_state *ifs,
57 	return test_bit(block, ifs->state);
61 		struct iomap_folio_state *ifs, size_t off, size_t len)
68 	bitmap_set(ifs->state, first_blk, nr_blks);
69 	return ifs_is_fully_uptodate(folio, ifs);
75 	struct iomap_folio_state *ifs = folio->private;
79 	if (ifs) {
80 		spin_lock_irqsave(&ifs->state_lock, flags);
81 		uptodate = ifs_set_range_uptodate(folio, ifs, off, len);
82 		spin_unlock_irqrestore(&ifs->state_lock, flags);
90 		struct iomap_folio_state *ifs, int block)
95 	return test_bit(block + blks_per_folio, ifs->state);
99 		struct iomap_folio_state *ifs, u64 *range_start, u64 range_end)
109 	while (!ifs_block_is_dirty(folio, ifs, start_blk))
114 		if (!ifs_block_is_dirty(folio, ifs, start_blk + nblks))
126 	struct iomap_folio_state *ifs = folio->private;
131 	if (ifs)
132 		return ifs_find_dirty_range(folio, ifs, range_start, range_end);
137 		struct iomap_folio_state *ifs, size_t off, size_t len)
146 	spin_lock_irqsave(&ifs->state_lock, flags);
147 	bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks);
148 	spin_unlock_irqrestore(&ifs->state_lock, flags);
153 	struct iomap_folio_state *ifs = folio->private;
155 	if (ifs)
156 		ifs_clear_range_dirty(folio, ifs, off, len);
160 		struct iomap_folio_state *ifs, size_t off, size_t len)
169 	spin_lock_irqsave(&ifs->state_lock, flags);
170 	bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks);
171 	spin_unlock_irqrestore(&ifs->state_lock, flags);
176 	struct iomap_folio_state *ifs = folio->private;
178 	if (ifs)
179 		ifs_set_range_dirty(folio, ifs, off, len);
185 	struct iomap_folio_state *ifs = folio->private;
189 	if (ifs || nr_blocks <= 1)
190 		return ifs;
198 	 * ifs->state tracks two sets of state flags when the
203 	ifs = kzalloc(struct_size(ifs, state,
205 	if (!ifs)
206 		return ifs;
208 	spin_lock_init(&ifs->state_lock);
210 		bitmap_set(ifs->state, 0, nr_blocks);
212 		bitmap_set(ifs->state, nr_blocks, nr_blocks);
213 	folio_attach_private(folio, ifs);
215 	return ifs;
220 	struct iomap_folio_state *ifs = folio_detach_private(folio);
222 	if (!ifs)
224 	WARN_ON_ONCE(ifs->read_bytes_pending != 0);
225 	WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending));
226 	WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=
228 	kfree(ifs);
237 	struct iomap_folio_state *ifs = folio->private;
252 	if (ifs) {
257 			if (!ifs_block_is_uptodate(ifs, i))
267 			if (ifs_block_is_uptodate(ifs, i)) {
294 	struct iomap_folio_state *ifs = folio->private;
298 	if (ifs) {
301 		spin_lock_irqsave(&ifs->state_lock, flags);
303 			uptodate = ifs_set_range_uptodate(folio, ifs, off, len);
304 		ifs->read_bytes_pending -= len;
305 		finished = !ifs->read_bytes_pending;
306 		spin_unlock_irqrestore(&ifs->state_lock, flags);
378 	struct iomap_folio_state *ifs;
387 	ifs = ifs_alloc(iter->inode, folio, iter->flags);
399 	if (ifs) {
400 		spin_lock_irq(&ifs->state_lock);
401 		ifs->read_bytes_pending += plen;
402 		spin_unlock_irq(&ifs->state_lock);
557 	struct iomap_folio_state *ifs = folio->private;
561 	if (!ifs)
572 		if (!ifs_block_is_uptodate(ifs, i))
675 	struct iomap_folio_state *ifs;
694 	ifs = ifs_alloc(iter->inode, folio, iter->flags);
695 	if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1)
1043 	struct iomap_folio_state *ifs;
1052 	ifs = folio->private;
1053 	if (!ifs)
1061 		if (!ifs_block_is_dirty(folio, ifs, i)) {
1512 	struct iomap_folio_state *ifs = folio->private;
1514 	WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);
1515 	WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0);
1517 	if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending))
1760 	struct iomap_folio_state *ifs = folio->private;
1775 	if (ifs)
1776 		atomic_add(len, &ifs->write_bytes_pending);
1894 	struct iomap_folio_state *ifs = folio->private;
1915 		if (!ifs) {
1916 			ifs = ifs_alloc(inode, folio, 0);
1923 		 * ifs->write_bytes_pending, which is dropped after submitting
1926 		WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending) != 0);
1927 		atomic_inc(&ifs->write_bytes_pending);
1965 	if (ifs) {
1966 		if (atomic_dec_and_test(&ifs->write_bytes_pending))