1/*	$NetBSD: uvm_object.c,v 1.25 2020/08/15 07:24:09 chs Exp $	*/
2
3/*
4 * Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Mindaugas Rasiukevicius.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * uvm_object.c: operate with memory objects
34 *
35 * TODO:
36 *  1. Support PG_RELEASED-using objects
37 */
38
39#include <sys/cdefs.h>
40__KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.25 2020/08/15 07:24:09 chs Exp $");
41
42#ifdef _KERNEL_OPT
43#include "opt_ddb.h"
44#endif
45
46#include <sys/param.h>
47#include <sys/rwlock.h>
48#include <sys/queue.h>
49
50#include <uvm/uvm.h>
51#include <uvm/uvm_ddb.h>
52#include <uvm/uvm_page_array.h>
53
54/* Page count to fetch per single step. */
55#define	FETCH_PAGECOUNT			16
56
57/*
58 * uvm_obj_init: initialize UVM memory object.
59 */
60void
61uvm_obj_init(struct uvm_object *uo, const struct uvm_pagerops *ops,
62    bool alock, u_int refs)
63{
64
65#if 0 /* notyet */
66	KASSERT(ops);
67#endif
68	if (alock) {
69		/* Allocate and assign a lock. */
70		uo->vmobjlock = rw_obj_alloc();
71	} else {
72		/* The lock will need to be set via uvm_obj_setlock(). */
73		uo->vmobjlock = NULL;
74	}
75	uo->pgops = ops;
76	LIST_INIT(&uo->uo_ubc);
77	uo->uo_npages = 0;
78	uo->uo_refs = refs;
79	radix_tree_init_tree(&uo->uo_pages);
80}
81
82/*
83 * uvm_obj_destroy: destroy UVM memory object.
84 */
85void
86uvm_obj_destroy(struct uvm_object *uo, bool dlock)
87{
88
89	KASSERT(radix_tree_empty_tree_p(&uo->uo_pages));
90
91	/* Purge any UBC entries associated with this object. */
92	ubc_purge(uo);
93
94	/* Destroy the lock, if requested. */
95	if (dlock) {
96		rw_obj_free(uo->vmobjlock);
97	}
98	radix_tree_fini_tree(&uo->uo_pages);
99}
100
101/*
102 * uvm_obj_setlock: assign a vmobjlock to the UVM object.
103 *
104 * => Caller is responsible to ensure that UVM objects is not use.
105 * => Only dynamic lock may be previously set.  We drop the reference then.
106 */
107void
108uvm_obj_setlock(struct uvm_object *uo, krwlock_t *lockptr)
109{
110	krwlock_t *olockptr = uo->vmobjlock;
111
112	if (olockptr) {
113		/* Drop the reference on the old lock. */
114		rw_obj_free(olockptr);
115	}
116	if (lockptr == NULL) {
117		/* If new lock is not passed - allocate default one. */
118		lockptr = rw_obj_alloc();
119	}
120	uo->vmobjlock = lockptr;
121}
122
123/*
124 * uvm_obj_wirepages: wire the pages of entire UVM object.
125 *
126 * => NOTE: this function should only be used for types of objects
127 *  where PG_RELEASED flag is never set (aobj objects)
128 * => caller must pass page-aligned start and end values
129 */
130int
131uvm_obj_wirepages(struct uvm_object *uobj, off_t start, off_t end,
132    struct pglist *list)
133{
134	int i, npages, error;
135	struct vm_page *pgs[FETCH_PAGECOUNT], *pg = NULL;
136	off_t offset = start, left;
137
138	left = (end - start) >> PAGE_SHIFT;
139
140	rw_enter(uobj->vmobjlock, RW_WRITER);
141	while (left) {
142
143		npages = MIN(FETCH_PAGECOUNT, left);
144
145		/* Get the pages */
146		memset(pgs, 0, sizeof(pgs));
147		error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0,
148			VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL,
149			PGO_SYNCIO);
150
151		if (error)
152			goto error;
153
154		rw_enter(uobj->vmobjlock, RW_WRITER);
155		for (i = 0; i < npages; i++) {
156
157			KASSERT(pgs[i] != NULL);
158			KASSERT(!(pgs[i]->flags & PG_RELEASED));
159
160			/*
161			 * Loan break
162			 */
163			if (pgs[i]->loan_count) {
164				while (pgs[i]->loan_count) {
165					pg = uvm_loanbreak(pgs[i]);
166					if (!pg) {
167						rw_exit(uobj->vmobjlock);
168						uvm_wait("uobjwirepg");
169						rw_enter(uobj->vmobjlock, RW_WRITER);
170						continue;
171					}
172				}
173				pgs[i] = pg;
174			}
175
176			if (pgs[i]->flags & PG_AOBJ) {
177				uvm_pagemarkdirty(pgs[i],
178				    UVM_PAGE_STATUS_DIRTY);
179				uao_dropswap(uobj, i);
180			}
181		}
182
183		/* Wire the pages */
184		for (i = 0; i < npages; i++) {
185			uvm_pagelock(pgs[i]);
186			uvm_pagewire(pgs[i]);
187			uvm_pageunlock(pgs[i]);
188			if (list != NULL)
189				TAILQ_INSERT_TAIL(list, pgs[i], pageq.queue);
190		}
191
192		/* Unbusy the pages */
193		uvm_page_unbusy(pgs, npages);
194
195		left -= npages;
196		offset += npages << PAGE_SHIFT;
197	}
198	rw_exit(uobj->vmobjlock);
199
200	return 0;
201
202error:
203	/* Unwire the pages which has been wired */
204	uvm_obj_unwirepages(uobj, start, offset);
205
206	return error;
207}
208
209/*
210 * uvm_obj_unwirepages: unwire the pages of entire UVM object.
211 *
212 * => NOTE: this function should only be used for types of objects
213 *  where PG_RELEASED flag is never set
214 * => caller must pass page-aligned start and end values
215 */
216void
217uvm_obj_unwirepages(struct uvm_object *uobj, off_t start, off_t end)
218{
219	struct vm_page *pg;
220	off_t offset;
221
222	rw_enter(uobj->vmobjlock, RW_WRITER);
223	for (offset = start; offset < end; offset += PAGE_SIZE) {
224		pg = uvm_pagelookup(uobj, offset);
225
226		KASSERT(pg != NULL);
227		KASSERT(!(pg->flags & PG_RELEASED));
228
229		uvm_pagelock(pg);
230		uvm_pageunwire(pg);
231		uvm_pageunlock(pg);
232	}
233	rw_exit(uobj->vmobjlock);
234}
235
236static inline bool
237uvm_obj_notag_p(struct uvm_object *uobj, int tag)
238{
239
240	KASSERT(rw_lock_held(uobj->vmobjlock));
241	return radix_tree_empty_tagged_tree_p(&uobj->uo_pages, tag);
242}
243
244bool
245uvm_obj_clean_p(struct uvm_object *uobj)
246{
247
248	return uvm_obj_notag_p(uobj, UVM_PAGE_DIRTY_TAG);
249}
250
251bool
252uvm_obj_nowriteback_p(struct uvm_object *uobj)
253{
254
255	return uvm_obj_notag_p(uobj, UVM_PAGE_WRITEBACK_TAG);
256}
257
258static inline bool
259uvm_obj_page_tag_p(struct vm_page *pg, int tag)
260{
261	struct uvm_object *uobj = pg->uobject;
262	uint64_t pgidx = pg->offset >> PAGE_SHIFT;
263
264	KASSERT(uobj != NULL);
265	KASSERT(rw_lock_held(uobj->vmobjlock));
266	return radix_tree_get_tag(&uobj->uo_pages, pgidx, tag) != 0;
267}
268
269static inline void
270uvm_obj_page_set_tag(struct vm_page *pg, int tag)
271{
272	struct uvm_object *uobj = pg->uobject;
273	uint64_t pgidx = pg->offset >> PAGE_SHIFT;
274
275	KASSERT(uobj != NULL);
276	KASSERT(rw_write_held(uobj->vmobjlock));
277	radix_tree_set_tag(&uobj->uo_pages, pgidx, tag);
278}
279
280static inline void
281uvm_obj_page_clear_tag(struct vm_page *pg, int tag)
282{
283	struct uvm_object *uobj = pg->uobject;
284	uint64_t pgidx = pg->offset >> PAGE_SHIFT;
285
286	KASSERT(uobj != NULL);
287	KASSERT(rw_write_held(uobj->vmobjlock));
288	radix_tree_clear_tag(&uobj->uo_pages, pgidx, tag);
289}
290
291bool
292uvm_obj_page_dirty_p(struct vm_page *pg)
293{
294
295	return uvm_obj_page_tag_p(pg, UVM_PAGE_DIRTY_TAG);
296}
297
298void
299uvm_obj_page_set_dirty(struct vm_page *pg)
300{
301
302	uvm_obj_page_set_tag(pg, UVM_PAGE_DIRTY_TAG);
303}
304
305void
306uvm_obj_page_clear_dirty(struct vm_page *pg)
307{
308
309	uvm_obj_page_clear_tag(pg, UVM_PAGE_DIRTY_TAG);
310}
311
312bool
313uvm_obj_page_writeback_p(struct vm_page *pg)
314{
315
316	return uvm_obj_page_tag_p(pg, UVM_PAGE_WRITEBACK_TAG);
317}
318
319void
320uvm_obj_page_set_writeback(struct vm_page *pg)
321{
322
323	uvm_obj_page_set_tag(pg, UVM_PAGE_WRITEBACK_TAG);
324}
325
326void
327uvm_obj_page_clear_writeback(struct vm_page *pg)
328{
329
330	uvm_obj_page_clear_tag(pg, UVM_PAGE_WRITEBACK_TAG);
331}
332
333#if defined(DDB) || defined(DEBUGPRINT)
334
335/*
336 * uvm_object_printit: actually prints the object
337 */
338void
339uvm_object_printit(struct uvm_object *uobj, bool full,
340    void (*pr)(const char *, ...))
341{
342	struct uvm_page_array a;
343	struct vm_page *pg;
344	int cnt = 0;
345	voff_t off;
346
347	(*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
348	    uobj, rw_write_held(uobj->vmobjlock), uobj->pgops, uobj->uo_npages);
349	if (UVM_OBJ_IS_KERN_OBJECT(uobj))
350		(*pr)("refs=<SYSTEM>\n");
351	else
352		(*pr)("refs=%d\n", uobj->uo_refs);
353
354	if (!full) {
355		return;
356	}
357	(*pr)("  PAGES <pg,offset>:\n  ");
358	uvm_page_array_init(&a, uobj, 0);
359	off = 0;
360	while ((pg = uvm_page_array_fill_and_peek(&a, off, 0)) != NULL) {
361		cnt++;
362		(*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
363		if ((cnt % 3) == 0) {
364			(*pr)("\n  ");
365		}
366		off = pg->offset + PAGE_SIZE;
367		uvm_page_array_advance(&a);
368	}
369	if ((cnt % 3) != 0) {
370		(*pr)("\n");
371	}
372	uvm_page_array_fini(&a);
373}
374
375#endif /* DDB || DEBUGPRINT */
376