linux-master/mm/filemap.c

127 static void mapping_set_update(struct xa_state *xas,
132 	xas_set_update(xas, workingset_update_node);
133 	xas_set_lru(xas, &shadow_nodes);
139 	XA_STATE(xas, &mapping->i_pages, folio->index);
142 	mapping_set_update(&xas, mapping);
144 	xas_set_order(&xas, folio->index, folio_order(folio));
149 	xas_store(&xas, shadow);
150 	xas_init_marks(&xas);
289 	XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);
294 	mapping_set_update(&xas, mapping);
295 	xas_for_each(&xas, folio, ULONG_MAX) {
321 		xas_store(&xas, NULL);
483 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
491 		folio = xas_find(&xas, max);
492 		if (xas_retry(&xas, folio))
641 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
649 	xas_for_each(&xas, folio, max) {
650 		if (xas_retry(&xas, folio))
819 	XA_STATE(xas, &mapping->i_pages, offset);
831 	xas_lock_irq(&xas);
832 	xas_store(&xas, new);
844 	xas_unlock_irq(&xas);
854 	XA_STATE(xas, &mapping->i_pages, index);
861 	mapping_set_update(&xas, mapping);
871 	xas_set_order(&xas, index, folio_order(folio));
877 	folio->index = xas.xa_index;
880 		unsigned int order = xa_get_order(xas.xa, xas.xa_index);
884 			xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),
886 		xas_lock_irq(&xas);
887 		xas_for_each_conflict(&xas, entry) {
890 				xas_set_err(&xas, -EEXIST);
899 			order = xa_get_order(xas.xa, xas.xa_index);
903 				xas_split(&xas, old, order);
904 				xas_reset(&xas);
908 		xas_store(&xas, folio);
909 		if (xas_error(&xas))
922 		xas_unlock_irq(&xas);
923 	} while (xas_nomem(&xas, gfp));
925 	if (xas_error(&xas))
936 	return xas_error(&xas);
1726 	XA_STATE(xas, &mapping->i_pages, index);
1729 		void *entry = xas_next(&xas);
1732 		if (xas.xa_index == 0)
1736 	return xas.xa_index;
1762 	XA_STATE(xas, &mapping->i_pages, index);
1765 		void *entry = xas_prev(&xas);
1768 		if (xas.xa_index == ULONG_MAX)
1772 	return xas.xa_index;
1810 	XA_STATE(xas, &mapping->i_pages, index);
1815 	xas_reset(&xas);
1816 	folio = xas_load(&xas);
1817 	if (xas_retry(&xas, folio))
1829 	if (unlikely(folio != xas_reload(&xas))) {
1959 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
1966 		folio = xas_find(xas, max);
1968 		folio = xas_find_marked(xas, max, mark);
1970 	if (xas_retry(xas, folio))
1983 	if (unlikely(folio != xas_reload(xas))) {
1990 	xas_reset(xas);
2017 	XA_STATE(xas, &mapping->i_pages, *start);
2021 	while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) {
2022 		indices[fbatch->nr] = xas.xa_index;
2063 	XA_STATE(xas, &mapping->i_pages, *start);
2067 	while ((folio = find_get_entry(&xas, end, XA_PRESENT))) {
2078 			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
2081 		indices[fbatch->nr] = xas.xa_index;
2143 	XA_STATE(xas, &mapping->i_pages, *start);
2149 	for (folio = xas_load(&xas); folio && xas.xa_index <= end;
2150 			folio = xas_next(&xas)) {
2151 		if (xas_retry(&xas, folio))
2163 		if (unlikely(folio != xas_reload(&xas)))
2176 		xas_reset(&xas);
2214 	XA_STATE(xas, &mapping->i_pages, *start);
2218 	while ((folio = find_get_entry(&xas, end, tag)) != NULL) {
2281 	XA_STATE(xas, &mapping->i_pages, index);
2285 	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
2286 		if (xas_retry(&xas, folio))
2288 		if (xas.xa_index > max || xa_is_value(folio))
2295 		if (unlikely(folio != xas_reload(&xas)))
2304 		xas_advance(&xas, folio_next_index(folio) - 1);
2309 		xas_reset(&xas);
2932 static inline loff_t folio_seek_hole_data(struct xa_state *xas,
2944 	xas_pause(xas);
2965 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
2968 		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
2993 	XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);
3002 	while ((folio = find_get_entry(&xas, max, XA_PRESENT))) {
3003 		loff_t pos = (u64)xas.xa_index << PAGE_SHIFT;
3012 		seek_size = seek_folio_size(&xas, folio);
3014 		start = folio_seek_hole_data(&xas, mapping, folio, start, pos,
3021 			xas_set(&xas, pos >> PAGE_SHIFT);
3436 static struct folio *next_uptodate_folio(struct xa_state *xas,
3439 	struct folio *folio = xas_next_entry(xas, end_pgoff);
3445 		if (xas_retry(xas, folio))
3454 		if (unlikely(folio != xas_reload(xas)))
3465 		if (xas->xa_index >= max_idx)
3472 	} while ((folio = xas_next_entry(xas, end_pgoff)) != NULL);
3571 	XA_STATE(xas, &mapping->i_pages, start_pgoff);
3577 	folio = next_uptodate_folio(&xas, mapping, end_pgoff);
3596 		addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
3597 		vmf->pte += xas.xa_index - last_pgoff;
3598 		last_pgoff = xas.xa_index;
3600 		nr_pages = min(end, end_pgoff) - xas.xa_index + 1;
3607 					xas.xa_index - folio->index, addr,
3612 	} while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL);
4153 	XA_STATE(xas, &mapping->i_pages, first_index);
4157 	xas_for_each(&xas, folio, last_index) {
4173 		if (xas_retry(&xas, folio))
4176 		order = xa_get_order(xas.xa, xas.xa_index);
4178 		folio_first_index = round_down(xas.xa_index, 1 << order);
4228 		if (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY))
4231 		if (xas_get_mark(&xas, PAGECACHE_TAG_WRITEBACK))
4236 			xas_pause(&xas);