sys/vm/vm_domainset.c

64 vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
68 	di->di_domain = ds;
69 	di->di_iter = iter;
70 	di->di_policy = ds->ds_policy;
71 	DOMAINSET_COPY(&ds->ds_mask, &di->di_valid_mask);
72 	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
90 		di->di_offset = pindex;
93 	di->di_minskip = true;
97 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
100 	*domain = di->di_domain->ds_order[
101 	    ++(*di->di_iter) % di->di_domain->ds_cnt];
105 vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
110 		d = di->di_domain->ds_order[
111 		    ++(*di->di_iter) % di->di_domain->ds_cnt];
112 	} while (d == di->di_domain->ds_prefer);
117 vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)
121 	d = di->di_offset % di->di_domain->ds_cnt;
122 	*di->di_iter = d;
123 	*domain = di->di_domain->ds_order[d];
127 vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
130 	KASSERT(di->di_n > 0,
131 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));
132 	switch (di->di_policy) {
142 		vm_domainset_iter_rr(di, domain);
145 		vm_domainset_iter_prefer(di, domain);
149 		    di->di_policy);
156 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
159 	switch (di->di_policy) {
162 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask)) {
167 			di->di_n = di->di_domain->ds_cnt + 1;
176 		di->di_n = di->di_domain->ds_cnt;
177 		vm_domainset_iter_rr(di, domain);
180 		*domain = di->di_domain->ds_prefer;
181 		di->di_n = di->di_domain->ds_cnt;
184 		vm_domainset_iter_interleave(di, domain);
185 		di->di_n = di->di_domain->ds_cnt;
189 		    di->di_policy);
191 	KASSERT(di->di_n > 0,
192 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));
198 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
212 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);
213 	di->di_flags = *req;
214 	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
216 	vm_domainset_iter_first(di, domain);
218 		vm_domainset_iter_page(di, obj, domain);
222 vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
225 	if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))
229 	while (--di->di_n != 0) {
230 		vm_domainset_iter_next(di, domain);
231 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
232 		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))
237 	if (di->di_minskip) {
238 		di->di_minskip = false;
239 		vm_domainset_iter_first(di, domain);
244 	if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
250 	vm_wait_doms(&di->di_valid_mask, 0);
253 	if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)
257 	vm_domainset_iter_first(di, domain);
263 _vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,
267 	di->di_flags = *flags;
268 	*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
269 	vm_domainset_iter_first(di, domain);
271 		vm_domainset_iter_policy(di, domain);
275 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
279 	vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);
280 	_vm_domainset_iter_policy_init(di, domain, flags);
284 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
288 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);
289 	_vm_domainset_iter_policy_init(di, domain, flags);
293 vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
295 	if (DOMAINSET_EMPTY(&di->di_valid_mask))
299 	while (--di->di_n != 0) {
300 		vm_domainset_iter_next(di, domain);
301 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
302 		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))
307 	if (di->di_minskip) {
308 		di->di_minskip = false;
309 		vm_domainset_iter_first(di, domain);
314 	if ((di->di_flags & M_WAITOK) == 0)
318 	vm_wait_doms(&di->di_valid_mask, 0);
321 	vm_domainset_iter_first(di, domain);
327 vm_domainset_iter_ignore(struct vm_domainset_iter *di, int domain)
329 	KASSERT(DOMAINSET_ISSET(domain, &di->di_valid_mask),
330 	    ("%s: domain %d not present in di_valid_mask for di %p",
331 	    __func__, domain, di));
332 	DOMAINSET_CLR(domain, &di->di_valid_mask);
338 vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
346 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
354 vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
361 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
369 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
377 vm_domainset_iter_ignore(struct vm_domainset_iter *di __unused,