1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/slab.h>
4#include "ctree.h"
5#include "subpage.h"
6#include "btrfs_inode.h"
7
8/*
9 * Subpage (sectorsize < PAGE_SIZE) support overview:
10 *
11 * Limitations:
12 *
13 * - Only support 64K page size for now
14 *   This is to make metadata handling easier, as 64K page would ensure
15 *   all nodesize would fit inside one page, thus we don't need to handle
16 *   cases where a tree block crosses several pages.
17 *
18 * - Only metadata read-write for now
19 *   The data read-write part is in development.
20 *
21 * - Metadata can't cross 64K page boundary
22 *   btrfs-progs and kernel have done that for a while, thus only ancient
23 *   filesystems could have such problem.  For such case, do a graceful
24 *   rejection.
25 *
26 * Special behavior:
27 *
28 * - Metadata
29 *   Metadata read is fully supported.
30 *   Meaning when reading one tree block will only trigger the read for the
31 *   needed range, other unrelated range in the same page will not be touched.
32 *
33 *   Metadata write support is partial.
34 *   The writeback is still for the full page, but we will only submit
35 *   the dirty extent buffers in the page.
36 *
37 *   This means, if we have a metadata page like this:
38 *
39 *   Page offset
40 *   0         16K         32K         48K        64K
41 *   |/////////|           |///////////|
42 *        \- Tree block A        \- Tree block B
43 *
44 *   Even if we just want to writeback tree block A, we will also writeback
45 *   tree block B if it's also dirty.
46 *
47 *   This may cause extra metadata writeback which results more COW.
48 *
49 * Implementation:
50 *
51 * - Common
52 *   Both metadata and data will use a new structure, btrfs_subpage, to
53 *   record the status of each sector inside a page.  This provides the extra
54 *   granularity needed.
55 *
56 * - Metadata
57 *   Since we have multiple tree blocks inside one page, we can't rely on page
58 *   locking anymore, or we will have greatly reduced concurrency or even
59 *   deadlocks (hold one tree lock while trying to lock another tree lock in
60 *   the same page).
61 *
62 *   Thus for metadata locking, subpage support relies on io_tree locking only.
63 *   This means a slightly higher tree locking latency.
64 */
65
66void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize)
67{
68	unsigned int cur = 0;
69	unsigned int nr_bits;
70
71	ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize));
72
73	nr_bits = PAGE_SIZE / sectorsize;
74	subpage_info->bitmap_nr_bits = nr_bits;
75
76	subpage_info->uptodate_offset = cur;
77	cur += nr_bits;
78
79	subpage_info->error_offset = cur;
80	cur += nr_bits;
81
82	subpage_info->dirty_offset = cur;
83	cur += nr_bits;
84
85	subpage_info->writeback_offset = cur;
86	cur += nr_bits;
87
88	subpage_info->ordered_offset = cur;
89	cur += nr_bits;
90
91	subpage_info->checked_offset = cur;
92	cur += nr_bits;
93
94	subpage_info->total_nr_bits = cur;
95}
96
97int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
98			 struct page *page, enum btrfs_subpage_type type)
99{
100	struct btrfs_subpage *subpage;
101
102	/*
103	 * We have cases like a dummy extent buffer page, which is not mappped
104	 * and doesn't need to be locked.
105	 */
106	if (page->mapping)
107		ASSERT(PageLocked(page));
108
109	/* Either not subpage, or the page already has private attached */
110	if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page))
111		return 0;
112
113	subpage = btrfs_alloc_subpage(fs_info, type);
114	if (IS_ERR(subpage))
115		return  PTR_ERR(subpage);
116
117	attach_page_private(page, subpage);
118	return 0;
119}
120
121void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info,
122			  struct page *page)
123{
124	struct btrfs_subpage *subpage;
125
126	/* Either not subpage, or already detached */
127	if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page))
128		return;
129
130	subpage = (struct btrfs_subpage *)detach_page_private(page);
131	ASSERT(subpage);
132	btrfs_free_subpage(subpage);
133}
134
135struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
136					  enum btrfs_subpage_type type)
137{
138	struct btrfs_subpage *ret;
139	unsigned int real_size;
140
141	ASSERT(fs_info->sectorsize < PAGE_SIZE);
142
143	real_size = struct_size(ret, bitmaps,
144			BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits));
145	ret = kzalloc(real_size, GFP_NOFS);
146	if (!ret)
147		return ERR_PTR(-ENOMEM);
148
149	spin_lock_init(&ret->lock);
150	if (type == BTRFS_SUBPAGE_METADATA) {
151		atomic_set(&ret->eb_refs, 0);
152	} else {
153		atomic_set(&ret->readers, 0);
154		atomic_set(&ret->writers, 0);
155	}
156	return ret;
157}
158
159void btrfs_free_subpage(struct btrfs_subpage *subpage)
160{
161	kfree(subpage);
162}
163
164/*
165 * Increase the eb_refs of current subpage.
166 *
167 * This is important for eb allocation, to prevent race with last eb freeing
168 * of the same page.
169 * With the eb_refs increased before the eb inserted into radix tree,
170 * detach_extent_buffer_page() won't detach the page private while we're still
171 * allocating the extent buffer.
172 */
173void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
174			    struct page *page)
175{
176	struct btrfs_subpage *subpage;
177
178	if (fs_info->sectorsize == PAGE_SIZE)
179		return;
180
181	ASSERT(PagePrivate(page) && page->mapping);
182	lockdep_assert_held(&page->mapping->private_lock);
183
184	subpage = (struct btrfs_subpage *)page->private;
185	atomic_inc(&subpage->eb_refs);
186}
187
188void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
189			    struct page *page)
190{
191	struct btrfs_subpage *subpage;
192
193	if (fs_info->sectorsize == PAGE_SIZE)
194		return;
195
196	ASSERT(PagePrivate(page) && page->mapping);
197	lockdep_assert_held(&page->mapping->private_lock);
198
199	subpage = (struct btrfs_subpage *)page->private;
200	ASSERT(atomic_read(&subpage->eb_refs));
201	atomic_dec(&subpage->eb_refs);
202}
203
204static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
205		struct page *page, u64 start, u32 len)
206{
207	/* Basic checks */
208	ASSERT(PagePrivate(page) && page->private);
209	ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
210	       IS_ALIGNED(len, fs_info->sectorsize));
211	/*
212	 * The range check only works for mapped page, we can still have
213	 * unmapped page like dummy extent buffer pages.
214	 */
215	if (page->mapping)
216		ASSERT(page_offset(page) <= start &&
217		       start + len <= page_offset(page) + PAGE_SIZE);
218}
219
220void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
221		struct page *page, u64 start, u32 len)
222{
223	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
224	const int nbits = len >> fs_info->sectorsize_bits;
225
226	btrfs_subpage_assert(fs_info, page, start, len);
227
228	atomic_add(nbits, &subpage->readers);
229}
230
231void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
232		struct page *page, u64 start, u32 len)
233{
234	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
235	const int nbits = len >> fs_info->sectorsize_bits;
236	bool is_data;
237	bool last;
238
239	btrfs_subpage_assert(fs_info, page, start, len);
240	is_data = is_data_inode(page->mapping->host);
241	ASSERT(atomic_read(&subpage->readers) >= nbits);
242	last = atomic_sub_and_test(nbits, &subpage->readers);
243
244	/*
245	 * For data we need to unlock the page if the last read has finished.
246	 *
247	 * And please don't replace @last with atomic_sub_and_test() call
248	 * inside if () condition.
249	 * As we want the atomic_sub_and_test() to be always executed.
250	 */
251	if (is_data && last)
252		unlock_page(page);
253}
254
255static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len)
256{
257	u64 orig_start = *start;
258	u32 orig_len = *len;
259
260	*start = max_t(u64, page_offset(page), orig_start);
261	/*
262	 * For certain call sites like btrfs_drop_pages(), we may have pages
263	 * beyond the target range. In that case, just set @len to 0, subpage
264	 * helpers can handle @len == 0 without any problem.
265	 */
266	if (page_offset(page) >= orig_start + orig_len)
267		*len = 0;
268	else
269		*len = min_t(u64, page_offset(page) + PAGE_SIZE,
270			     orig_start + orig_len) - *start;
271}
272
273void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,
274		struct page *page, u64 start, u32 len)
275{
276	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
277	const int nbits = (len >> fs_info->sectorsize_bits);
278	int ret;
279
280	btrfs_subpage_assert(fs_info, page, start, len);
281
282	ASSERT(atomic_read(&subpage->readers) == 0);
283	ret = atomic_add_return(nbits, &subpage->writers);
284	ASSERT(ret == nbits);
285}
286
287bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,
288		struct page *page, u64 start, u32 len)
289{
290	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
291	const int nbits = (len >> fs_info->sectorsize_bits);
292
293	btrfs_subpage_assert(fs_info, page, start, len);
294
295	/*
296	 * We have call sites passing @lock_page into
297	 * extent_clear_unlock_delalloc() for compression path.
298	 *
299	 * This @locked_page is locked by plain lock_page(), thus its
300	 * subpage::writers is 0.  Handle them in a special way.
301	 */
302	if (atomic_read(&subpage->writers) == 0)
303		return true;
304
305	ASSERT(atomic_read(&subpage->writers) >= nbits);
306	return atomic_sub_and_test(nbits, &subpage->writers);
307}
308
309/*
310 * Lock a page for delalloc page writeback.
311 *
312 * Return -EAGAIN if the page is not properly initialized.
313 * Return 0 with the page locked, and writer counter updated.
314 *
315 * Even with 0 returned, the page still need extra check to make sure
316 * it's really the correct page, as the caller is using
317 * find_get_pages_contig(), which can race with page invalidating.
318 */
319int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info,
320		struct page *page, u64 start, u32 len)
321{
322	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {
323		lock_page(page);
324		return 0;
325	}
326	lock_page(page);
327	if (!PagePrivate(page) || !page->private) {
328		unlock_page(page);
329		return -EAGAIN;
330	}
331	btrfs_subpage_clamp_range(page, &start, &len);
332	btrfs_subpage_start_writer(fs_info, page, start, len);
333	return 0;
334}
335
336void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info,
337		struct page *page, u64 start, u32 len)
338{
339	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)
340		return unlock_page(page);
341	btrfs_subpage_clamp_range(page, &start, &len);
342	if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len))
343		unlock_page(page);
344}
345
346static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start,
347				      unsigned int nbits)
348{
349	unsigned int found_zero;
350
351	found_zero = find_next_zero_bit(addr, start + nbits, start);
352	if (found_zero == start + nbits)
353		return true;
354	return false;
355}
356
357static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start,
358				       unsigned int nbits)
359{
360	unsigned int found_set;
361
362	found_set = find_next_bit(addr, start + nbits, start);
363	if (found_set == start + nbits)
364		return true;
365	return false;
366}
367
368#define subpage_calc_start_bit(fs_info, page, name, start, len)		\
369({									\
370	unsigned int start_bit;						\
371									\
372	btrfs_subpage_assert(fs_info, page, start, len);		\
373	start_bit = offset_in_page(start) >> fs_info->sectorsize_bits;	\
374	start_bit += fs_info->subpage_info->name##_offset;		\
375	start_bit;							\
376})
377
378#define subpage_test_bitmap_all_set(fs_info, subpage, name)		\
379	bitmap_test_range_all_set(subpage->bitmaps,			\
380			fs_info->subpage_info->name##_offset,		\
381			fs_info->subpage_info->bitmap_nr_bits)
382
383#define subpage_test_bitmap_all_zero(fs_info, subpage, name)		\
384	bitmap_test_range_all_zero(subpage->bitmaps,			\
385			fs_info->subpage_info->name##_offset,		\
386			fs_info->subpage_info->bitmap_nr_bits)
387
388void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
389		struct page *page, u64 start, u32 len)
390{
391	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
392	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
393							uptodate, start, len);
394	unsigned long flags;
395
396	spin_lock_irqsave(&subpage->lock, flags);
397	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
398	if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))
399		SetPageUptodate(page);
400	spin_unlock_irqrestore(&subpage->lock, flags);
401}
402
403void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
404		struct page *page, u64 start, u32 len)
405{
406	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
407	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
408							uptodate, start, len);
409	unsigned long flags;
410
411	spin_lock_irqsave(&subpage->lock, flags);
412	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
413	ClearPageUptodate(page);
414	spin_unlock_irqrestore(&subpage->lock, flags);
415}
416
417void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
418		struct page *page, u64 start, u32 len)
419{
420	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
421	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
422							error, start, len);
423	unsigned long flags;
424
425	spin_lock_irqsave(&subpage->lock, flags);
426	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
427	SetPageError(page);
428	spin_unlock_irqrestore(&subpage->lock, flags);
429}
430
431void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
432		struct page *page, u64 start, u32 len)
433{
434	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
435	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
436							error, start, len);
437	unsigned long flags;
438
439	spin_lock_irqsave(&subpage->lock, flags);
440	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
441	if (subpage_test_bitmap_all_zero(fs_info, subpage, error))
442		ClearPageError(page);
443	spin_unlock_irqrestore(&subpage->lock, flags);
444}
445
446void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
447		struct page *page, u64 start, u32 len)
448{
449	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
450	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
451							dirty, start, len);
452	unsigned long flags;
453
454	spin_lock_irqsave(&subpage->lock, flags);
455	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
456	spin_unlock_irqrestore(&subpage->lock, flags);
457	set_page_dirty(page);
458}
459
460/*
461 * Extra clear_and_test function for subpage dirty bitmap.
462 *
463 * Return true if we're the last bits in the dirty_bitmap and clear the
464 * dirty_bitmap.
465 * Return false otherwise.
466 *
467 * NOTE: Callers should manually clear page dirty for true case, as we have
468 * extra handling for tree blocks.
469 */
470bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
471		struct page *page, u64 start, u32 len)
472{
473	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
474	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
475							dirty, start, len);
476	unsigned long flags;
477	bool last = false;
478
479	spin_lock_irqsave(&subpage->lock, flags);
480	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
481	if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))
482		last = true;
483	spin_unlock_irqrestore(&subpage->lock, flags);
484	return last;
485}
486
487void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,
488		struct page *page, u64 start, u32 len)
489{
490	bool last;
491
492	last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len);
493	if (last)
494		clear_page_dirty_for_io(page);
495}
496
497void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
498		struct page *page, u64 start, u32 len)
499{
500	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
501	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
502							writeback, start, len);
503	unsigned long flags;
504
505	spin_lock_irqsave(&subpage->lock, flags);
506	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
507	set_page_writeback(page);
508	spin_unlock_irqrestore(&subpage->lock, flags);
509}
510
511void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
512		struct page *page, u64 start, u32 len)
513{
514	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
515	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
516							writeback, start, len);
517	unsigned long flags;
518
519	spin_lock_irqsave(&subpage->lock, flags);
520	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
521	if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {
522		ASSERT(PageWriteback(page));
523		end_page_writeback(page);
524	}
525	spin_unlock_irqrestore(&subpage->lock, flags);
526}
527
528void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
529		struct page *page, u64 start, u32 len)
530{
531	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
532	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
533							ordered, start, len);
534	unsigned long flags;
535
536	spin_lock_irqsave(&subpage->lock, flags);
537	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
538	SetPageOrdered(page);
539	spin_unlock_irqrestore(&subpage->lock, flags);
540}
541
542void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
543		struct page *page, u64 start, u32 len)
544{
545	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
546	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
547							ordered, start, len);
548	unsigned long flags;
549
550	spin_lock_irqsave(&subpage->lock, flags);
551	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
552	if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))
553		ClearPageOrdered(page);
554	spin_unlock_irqrestore(&subpage->lock, flags);
555}
556
557void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info,
558			       struct page *page, u64 start, u32 len)
559{
560	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
561	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
562							checked, start, len);
563	unsigned long flags;
564
565	spin_lock_irqsave(&subpage->lock, flags);
566	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
567	if (subpage_test_bitmap_all_set(fs_info, subpage, checked))
568		SetPageChecked(page);
569	spin_unlock_irqrestore(&subpage->lock, flags);
570}
571
572void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info,
573				 struct page *page, u64 start, u32 len)
574{
575	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
576	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
577							checked, start, len);
578	unsigned long flags;
579
580	spin_lock_irqsave(&subpage->lock, flags);
581	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
582	ClearPageChecked(page);
583	spin_unlock_irqrestore(&subpage->lock, flags);
584}
585
586/*
587 * Unlike set/clear which is dependent on each page status, for test all bits
588 * are tested in the same way.
589 */
590#define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name)				\
591bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info,	\
592		struct page *page, u64 start, u32 len)			\
593{									\
594	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
595	unsigned int start_bit = subpage_calc_start_bit(fs_info, page,	\
596						name, start, len);	\
597	unsigned long flags;						\
598	bool ret;							\
599									\
600	spin_lock_irqsave(&subpage->lock, flags);			\
601	ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit,	\
602				len >> fs_info->sectorsize_bits);	\
603	spin_unlock_irqrestore(&subpage->lock, flags);			\
604	return ret;							\
605}
606IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
607IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error);
608IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
609IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
610IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);
611IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked);
612
613/*
614 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
615 * in.  We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
616 * back to regular sectorsize branch.
617 */
618#define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func,	\
619			       test_page_func)				\
620void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info,		\
621		struct page *page, u64 start, u32 len)			\
622{									\
623	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {	\
624		set_page_func(page);					\
625		return;							\
626	}								\
627	btrfs_subpage_set_##name(fs_info, page, start, len);		\
628}									\
629void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info,	\
630		struct page *page, u64 start, u32 len)			\
631{									\
632	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {	\
633		clear_page_func(page);					\
634		return;							\
635	}								\
636	btrfs_subpage_clear_##name(fs_info, page, start, len);		\
637}									\
638bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info,	\
639		struct page *page, u64 start, u32 len)			\
640{									\
641	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)	\
642		return test_page_func(page);				\
643	return btrfs_subpage_test_##name(fs_info, page, start, len);	\
644}									\
645void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info,	\
646		struct page *page, u64 start, u32 len)			\
647{									\
648	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {	\
649		set_page_func(page);					\
650		return;							\
651	}								\
652	btrfs_subpage_clamp_range(page, &start, &len);			\
653	btrfs_subpage_set_##name(fs_info, page, start, len);		\
654}									\
655void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \
656		struct page *page, u64 start, u32 len)			\
657{									\
658	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {	\
659		clear_page_func(page);					\
660		return;							\
661	}								\
662	btrfs_subpage_clamp_range(page, &start, &len);			\
663	btrfs_subpage_clear_##name(fs_info, page, start, len);		\
664}									\
665bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info,	\
666		struct page *page, u64 start, u32 len)			\
667{									\
668	if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)	\
669		return test_page_func(page);				\
670	btrfs_subpage_clamp_range(page, &start, &len);			\
671	return btrfs_subpage_test_##name(fs_info, page, start, len);	\
672}
673IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate,
674			 PageUptodate);
675IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError);
676IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io,
677			 PageDirty);
678IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback,
679			 PageWriteback);
680IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered,
681			 PageOrdered);
682IMPLEMENT_BTRFS_PAGE_OPS(checked, SetPageChecked, ClearPageChecked, PageChecked);
683
684/*
685 * Make sure not only the page dirty bit is cleared, but also subpage dirty bit
686 * is cleared.
687 */
688void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info,
689				 struct page *page)
690{
691	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
692
693	if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
694		return;
695
696	ASSERT(!PageDirty(page));
697	if (fs_info->sectorsize == PAGE_SIZE)
698		return;
699
700	ASSERT(PagePrivate(page) && page->private);
701	ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty));
702}
703
704/*
705 * Handle different locked pages with different page sizes:
706 *
707 * - Page locked by plain lock_page()
708 *   It should not have any subpage::writers count.
709 *   Can be unlocked by unlock_page().
710 *   This is the most common locked page for __extent_writepage() called
711 *   inside extent_write_cache_pages() or extent_write_full_page().
712 *   Rarer cases include the @locked_page from extent_write_locked_range().
713 *
714 * - Page locked by lock_delalloc_pages()
715 *   There is only one caller, all pages except @locked_page for
716 *   extent_write_locked_range().
717 *   In this case, we have to call subpage helper to handle the case.
718 */
719void btrfs_page_unlock_writer(struct btrfs_fs_info *fs_info, struct page *page,
720			      u64 start, u32 len)
721{
722	struct btrfs_subpage *subpage;
723
724	ASSERT(PageLocked(page));
725	/* For regular page size case, we just unlock the page */
726	if (fs_info->sectorsize == PAGE_SIZE)
727		return unlock_page(page);
728
729	ASSERT(PagePrivate(page) && page->private);
730	subpage = (struct btrfs_subpage *)page->private;
731
732	/*
733	 * For subpage case, there are two types of locked page.  With or
734	 * without writers number.
735	 *
736	 * Since we own the page lock, no one else could touch subpage::writers
737	 * and we are safe to do several atomic operations without spinlock.
738	 */
739	if (atomic_read(&subpage->writers))
740		/* No writers, locked by plain lock_page() */
741		return unlock_page(page);
742
743	/* Have writers, use proper subpage helper to end it */
744	btrfs_page_end_writer_lock(fs_info, page, start, len);
745}
746