1/*-
2 * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer,
10 *    without modification.
11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12 *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
13 *    redistribution must be conditioned upon including a substantially
14 *    similar Disclaimer requirement for further binary redistribution.
15 *
16 * NO WARRANTY
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
20 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
22 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
27 * THE POSSIBILITY OF SUCH DAMAGES.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include "opt_vm.h"
34#include "opt_ddb.h"
35
36#include <sys/param.h>
37#include <sys/systm.h>
38#include <sys/lock.h>
39#include <sys/kernel.h>
40#include <sys/malloc.h>
41#include <sys/mutex.h>
42#ifdef VM_NUMA_ALLOC
43#include <sys/proc.h>
44#endif
45#include <sys/queue.h>
46#include <sys/rwlock.h>
47#include <sys/sbuf.h>
48#include <sys/sysctl.h>
49#include <sys/tree.h>
50#include <sys/vmmeter.h>
51#include <sys/seq.h>
52
53#include <ddb/ddb.h>
54
55#include <vm/vm.h>
56#include <vm/vm_param.h>
57#include <vm/vm_kern.h>
58#include <vm/vm_object.h>
59#include <vm/vm_page.h>
60#include <vm/vm_phys.h>
61
62#include <vm/vm_domain.h>
63
64#ifdef VM_NUMA_ALLOC
65static __inline int
66vm_domain_rr_selectdomain(int skip_domain)
67{
68	struct thread *td;
69
70	td = curthread;
71
72	td->td_dom_rr_idx++;
73	td->td_dom_rr_idx %= vm_ndomains;
74
75	/*
76	 * If skip_domain is provided then skip over that
77	 * domain.  This is intended for round robin variants
78	 * which first try a fixed domain.
79	 */
80	if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) {
81		td->td_dom_rr_idx++;
82		td->td_dom_rr_idx %= vm_ndomains;
83	}
84	return (td->td_dom_rr_idx);
85}
86#endif
87
88/*
89 * This implements a very simple set of VM domain memory allocation
90 * policies and iterators.
91 */
92
93/*
94 * A VM domain policy represents a desired VM domain policy.
95 * Iterators implement searching through VM domains in a specific
96 * order.
97 */
98
99/*
100 * When setting a policy, the caller must establish their own
101 * exclusive write protection for the contents of the domain
102 * policy.
103 */
104int
105vm_domain_policy_init(struct vm_domain_policy *vp)
106{
107
108	bzero(vp, sizeof(*vp));
109	vp->p.policy = VM_POLICY_NONE;
110	vp->p.domain = -1;
111	return (0);
112}
113
114int
115vm_domain_policy_set(struct vm_domain_policy *vp,
116    vm_domain_policy_type_t vt, int domain)
117{
118
119	seq_write_begin(&vp->seq);
120	vp->p.policy = vt;
121	vp->p.domain = domain;
122	seq_write_end(&vp->seq);
123	return (0);
124}
125
126/*
127 * Take a local copy of a policy.
128 *
129 * The destination policy isn't write-barriered; this is used
130 * for doing local copies into something that isn't shared.
131 */
132void
133vm_domain_policy_localcopy(struct vm_domain_policy *dst,
134    const struct vm_domain_policy *src)
135{
136	seq_t seq;
137
138	for (;;) {
139		seq = seq_read(&src->seq);
140		*dst = *src;
141		if (seq_consistent(&src->seq, seq))
142			return;
143		cpu_spinwait();
144	}
145}
146
147/*
148 * Take a write-barrier copy of a policy.
149 *
150 * The destination policy is write -barriered; this is used
151 * for doing copies into policies that may be read by other
152 * threads.
153 */
154void
155vm_domain_policy_copy(struct vm_domain_policy *dst,
156    const struct vm_domain_policy *src)
157{
158	seq_t seq;
159	struct vm_domain_policy d;
160
161	for (;;) {
162		seq = seq_read(&src->seq);
163		d = *src;
164		if (seq_consistent(&src->seq, seq)) {
165			seq_write_begin(&dst->seq);
166			dst->p.domain = d.p.domain;
167			dst->p.policy = d.p.policy;
168			seq_write_end(&dst->seq);
169			return;
170		}
171		cpu_spinwait();
172	}
173}
174
175int
176vm_domain_policy_validate(const struct vm_domain_policy *vp)
177{
178
179	switch (vp->p.policy) {
180	case VM_POLICY_NONE:
181	case VM_POLICY_ROUND_ROBIN:
182	case VM_POLICY_FIRST_TOUCH:
183	case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
184		if (vp->p.domain == -1)
185			return (0);
186		return (-1);
187	case VM_POLICY_FIXED_DOMAIN:
188	case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
189#ifdef VM_NUMA_ALLOC
190		if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains)
191			return (0);
192#else
193		if (vp->p.domain == 0)
194			return (0);
195#endif
196		return (-1);
197	default:
198		return (-1);
199	}
200	return (-1);
201}
202
203int
204vm_domain_policy_cleanup(struct vm_domain_policy *vp)
205{
206
207	/* For now, empty */
208	return (0);
209}
210
211int
212vm_domain_iterator_init(struct vm_domain_iterator *vi)
213{
214
215	/* Nothing to do for now */
216	return (0);
217}
218
219/*
220 * Manually setup an iterator with the given details.
221 */
222int
223vm_domain_iterator_set(struct vm_domain_iterator *vi,
224    vm_domain_policy_type_t vt, int domain)
225{
226
227#ifdef VM_NUMA_ALLOC
228	switch (vt) {
229	case VM_POLICY_FIXED_DOMAIN:
230		vi->policy = VM_POLICY_FIXED_DOMAIN;
231		vi->domain = domain;
232		vi->n = 1;
233		break;
234	case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
235		vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN;
236		vi->domain = domain;
237		vi->n = vm_ndomains;
238		break;
239	case VM_POLICY_FIRST_TOUCH:
240		vi->policy = VM_POLICY_FIRST_TOUCH;
241		vi->domain = PCPU_GET(domain);
242		vi->n = 1;
243		break;
244	case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
245		vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN;
246		vi->domain = PCPU_GET(domain);
247		vi->n = vm_ndomains;
248		break;
249	case VM_POLICY_ROUND_ROBIN:
250	default:
251		vi->policy = VM_POLICY_ROUND_ROBIN;
252		vi->domain = -1;
253		vi->n = vm_ndomains;
254		break;
255	}
256#else
257	vi->domain = 0;
258	vi->n = 1;
259#endif
260	return (0);
261}
262
263/*
264 * Setup an iterator based on the given policy.
265 */
266static inline void
267_vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
268    const struct vm_domain_policy *vt)
269{
270
271#ifdef VM_NUMA_ALLOC
272	/*
273	 * Initialise the iterator.
274	 *
275	 * For first-touch, the initial domain is set
276	 * via the current thread CPU domain.
277	 *
278	 * For fixed-domain, it's assumed that the
279	 * caller has initialised the specific domain
280	 * it is after.
281	 */
282	switch (vt->p.policy) {
283	case VM_POLICY_FIXED_DOMAIN:
284		vi->policy = vt->p.policy;
285		vi->domain = vt->p.domain;
286		vi->n = 1;
287		break;
288	case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
289		vi->policy = vt->p.policy;
290		vi->domain = vt->p.domain;
291		vi->n = vm_ndomains;
292		break;
293	case VM_POLICY_FIRST_TOUCH:
294		vi->policy = vt->p.policy;
295		vi->domain = PCPU_GET(domain);
296		vi->n = 1;
297		break;
298	case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
299		vi->policy = vt->p.policy;
300		vi->domain = PCPU_GET(domain);
301		vi->n = vm_ndomains;
302		break;
303	case VM_POLICY_ROUND_ROBIN:
304	default:
305		/*
306		 * Default to round-robin policy.
307		 */
308		vi->policy = VM_POLICY_ROUND_ROBIN;
309		vi->domain = -1;
310		vi->n = vm_ndomains;
311		break;
312	}
313#else
314	vi->domain = 0;
315	vi->n = 1;
316#endif
317}
318
319void
320vm_domain_iterator_set_policy(struct vm_domain_iterator *vi,
321    const struct vm_domain_policy *vt)
322{
323	seq_t seq;
324	struct vm_domain_policy vt_lcl;
325
326	for (;;) {
327		seq = seq_read(&vt->seq);
328		vt_lcl = *vt;
329		if (seq_consistent(&vt->seq, seq)) {
330			_vm_domain_iterator_set_policy(vi, &vt_lcl);
331			return;
332		}
333		cpu_spinwait();
334	}
335}
336
337/*
338 * Return the next VM domain to use.
339 *
340 * Returns 0 w/ domain set to the next domain to use, or
341 * -1 to indicate no more domains are available.
342 */
343int
344vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain)
345{
346
347	/* General catch-all */
348	if (vi->n <= 0)
349		return (-1);
350
351#ifdef VM_NUMA_ALLOC
352	switch (vi->policy) {
353	case VM_POLICY_FIXED_DOMAIN:
354	case VM_POLICY_FIRST_TOUCH:
355		*domain = vi->domain;
356		vi->n--;
357		break;
358	case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN:
359	case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN:
360		/*
361		 * XXX TODO: skip over the rr'ed domain
362		 * if it equals the one we started with.
363		 */
364		if (vi->n == vm_ndomains)
365			*domain = vi->domain;
366		else
367			*domain = vm_domain_rr_selectdomain(vi->domain);
368		vi->n--;
369		break;
370	case VM_POLICY_ROUND_ROBIN:
371	default:
372		*domain = vm_domain_rr_selectdomain(-1);
373		vi->n--;
374		break;
375	}
376#else
377	*domain = 0;
378	vi->n--;
379#endif
380
381	return (0);
382}
383
384/*
385 * Returns 1 if the iteration is done, or 0 if it has not.
386
387 * This can only be called after at least one loop through
388 * the iterator.  Ie, it's designed to be used as a tail
389 * check of a loop, not the head check of a loop.
390 */
391int
392vm_domain_iterator_isdone(struct vm_domain_iterator *vi)
393{
394
395	return (vi->n <= 0);
396}
397
398int
399vm_domain_iterator_cleanup(struct vm_domain_iterator *vi)
400{
401
402	return (0);
403}
404