1/*- 2 * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification. 11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 12 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 13 * redistribution must be conditioned upon including a substantially 14 * similar Disclaimer requirement for further binary redistribution. 15 * 16 * NO WARRANTY 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 20 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 22 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 27 * THE POSSIBILITY OF SUCH DAMAGES. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD$"); 32 33#include "opt_vm.h" 34#include "opt_ddb.h" 35 36#include <sys/param.h> 37#include <sys/systm.h> 38#include <sys/lock.h> 39#include <sys/kernel.h> 40#include <sys/malloc.h> 41#include <sys/mutex.h> 42#ifdef VM_NUMA_ALLOC 43#include <sys/proc.h> 44#endif 45#include <sys/queue.h> 46#include <sys/rwlock.h> 47#include <sys/sbuf.h> 48#include <sys/sysctl.h> 49#include <sys/tree.h> 50#include <sys/vmmeter.h> 51#include <sys/seq.h> 52 53#include <ddb/ddb.h> 54 55#include <vm/vm.h> 56#include <vm/vm_param.h> 57#include <vm/vm_kern.h> 58#include <vm/vm_object.h> 59#include <vm/vm_page.h> 60#include <vm/vm_phys.h> 61 62#include <vm/vm_domain.h> 63 64#ifdef VM_NUMA_ALLOC 65static __inline int 66vm_domain_rr_selectdomain(int skip_domain) 67{ 68 struct thread *td; 69 70 td = curthread; 71 72 td->td_dom_rr_idx++; 73 td->td_dom_rr_idx %= vm_ndomains; 74 75 /* 76 * If skip_domain is provided then skip over that 77 * domain. This is intended for round robin variants 78 * which first try a fixed domain. 79 */ 80 if ((skip_domain > -1) && (td->td_dom_rr_idx == skip_domain)) { 81 td->td_dom_rr_idx++; 82 td->td_dom_rr_idx %= vm_ndomains; 83 } 84 return (td->td_dom_rr_idx); 85} 86#endif 87 88/* 89 * This implements a very simple set of VM domain memory allocation 90 * policies and iterators. 91 */ 92 93/* 94 * A VM domain policy represents a desired VM domain policy. 95 * Iterators implement searching through VM domains in a specific 96 * order. 97 */ 98 99/* 100 * When setting a policy, the caller must establish their own 101 * exclusive write protection for the contents of the domain 102 * policy. 103 */ 104int 105vm_domain_policy_init(struct vm_domain_policy *vp) 106{ 107 108 bzero(vp, sizeof(*vp)); 109 vp->p.policy = VM_POLICY_NONE; 110 vp->p.domain = -1; 111 return (0); 112} 113 114int 115vm_domain_policy_set(struct vm_domain_policy *vp, 116 vm_domain_policy_type_t vt, int domain) 117{ 118 119 seq_write_begin(&vp->seq); 120 vp->p.policy = vt; 121 vp->p.domain = domain; 122 seq_write_end(&vp->seq); 123 return (0); 124} 125 126/* 127 * Take a local copy of a policy. 128 * 129 * The destination policy isn't write-barriered; this is used 130 * for doing local copies into something that isn't shared. 131 */ 132void 133vm_domain_policy_localcopy(struct vm_domain_policy *dst, 134 const struct vm_domain_policy *src) 135{ 136 seq_t seq; 137 138 for (;;) { 139 seq = seq_read(&src->seq); 140 *dst = *src; 141 if (seq_consistent(&src->seq, seq)) 142 return; 143 cpu_spinwait(); 144 } 145} 146 147/* 148 * Take a write-barrier copy of a policy. 149 * 150 * The destination policy is write -barriered; this is used 151 * for doing copies into policies that may be read by other 152 * threads. 153 */ 154void 155vm_domain_policy_copy(struct vm_domain_policy *dst, 156 const struct vm_domain_policy *src) 157{ 158 seq_t seq; 159 struct vm_domain_policy d; 160 161 for (;;) { 162 seq = seq_read(&src->seq); 163 d = *src; 164 if (seq_consistent(&src->seq, seq)) { 165 seq_write_begin(&dst->seq); 166 dst->p.domain = d.p.domain; 167 dst->p.policy = d.p.policy; 168 seq_write_end(&dst->seq); 169 return; 170 } 171 cpu_spinwait(); 172 } 173} 174 175int 176vm_domain_policy_validate(const struct vm_domain_policy *vp) 177{ 178 179 switch (vp->p.policy) { 180 case VM_POLICY_NONE: 181 case VM_POLICY_ROUND_ROBIN: 182 case VM_POLICY_FIRST_TOUCH: 183 case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN: 184 if (vp->p.domain == -1) 185 return (0); 186 return (-1); 187 case VM_POLICY_FIXED_DOMAIN: 188 case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN: 189#ifdef VM_NUMA_ALLOC 190 if (vp->p.domain >= 0 && vp->p.domain < vm_ndomains) 191 return (0); 192#else 193 if (vp->p.domain == 0) 194 return (0); 195#endif 196 return (-1); 197 default: 198 return (-1); 199 } 200 return (-1); 201} 202 203int 204vm_domain_policy_cleanup(struct vm_domain_policy *vp) 205{ 206 207 /* For now, empty */ 208 return (0); 209} 210 211int 212vm_domain_iterator_init(struct vm_domain_iterator *vi) 213{ 214 215 /* Nothing to do for now */ 216 return (0); 217} 218 219/* 220 * Manually setup an iterator with the given details. 221 */ 222int 223vm_domain_iterator_set(struct vm_domain_iterator *vi, 224 vm_domain_policy_type_t vt, int domain) 225{ 226 227#ifdef VM_NUMA_ALLOC 228 switch (vt) { 229 case VM_POLICY_FIXED_DOMAIN: 230 vi->policy = VM_POLICY_FIXED_DOMAIN; 231 vi->domain = domain; 232 vi->n = 1; 233 break; 234 case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN: 235 vi->policy = VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN; 236 vi->domain = domain; 237 vi->n = vm_ndomains; 238 break; 239 case VM_POLICY_FIRST_TOUCH: 240 vi->policy = VM_POLICY_FIRST_TOUCH; 241 vi->domain = PCPU_GET(domain); 242 vi->n = 1; 243 break; 244 case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN: 245 vi->policy = VM_POLICY_FIRST_TOUCH_ROUND_ROBIN; 246 vi->domain = PCPU_GET(domain); 247 vi->n = vm_ndomains; 248 break; 249 case VM_POLICY_ROUND_ROBIN: 250 default: 251 vi->policy = VM_POLICY_ROUND_ROBIN; 252 vi->domain = -1; 253 vi->n = vm_ndomains; 254 break; 255 } 256#else 257 vi->domain = 0; 258 vi->n = 1; 259#endif 260 return (0); 261} 262 263/* 264 * Setup an iterator based on the given policy. 265 */ 266static inline void 267_vm_domain_iterator_set_policy(struct vm_domain_iterator *vi, 268 const struct vm_domain_policy *vt) 269{ 270 271#ifdef VM_NUMA_ALLOC 272 /* 273 * Initialise the iterator. 274 * 275 * For first-touch, the initial domain is set 276 * via the current thread CPU domain. 277 * 278 * For fixed-domain, it's assumed that the 279 * caller has initialised the specific domain 280 * it is after. 281 */ 282 switch (vt->p.policy) { 283 case VM_POLICY_FIXED_DOMAIN: 284 vi->policy = vt->p.policy; 285 vi->domain = vt->p.domain; 286 vi->n = 1; 287 break; 288 case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN: 289 vi->policy = vt->p.policy; 290 vi->domain = vt->p.domain; 291 vi->n = vm_ndomains; 292 break; 293 case VM_POLICY_FIRST_TOUCH: 294 vi->policy = vt->p.policy; 295 vi->domain = PCPU_GET(domain); 296 vi->n = 1; 297 break; 298 case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN: 299 vi->policy = vt->p.policy; 300 vi->domain = PCPU_GET(domain); 301 vi->n = vm_ndomains; 302 break; 303 case VM_POLICY_ROUND_ROBIN: 304 default: 305 /* 306 * Default to round-robin policy. 307 */ 308 vi->policy = VM_POLICY_ROUND_ROBIN; 309 vi->domain = -1; 310 vi->n = vm_ndomains; 311 break; 312 } 313#else 314 vi->domain = 0; 315 vi->n = 1; 316#endif 317} 318 319void 320vm_domain_iterator_set_policy(struct vm_domain_iterator *vi, 321 const struct vm_domain_policy *vt) 322{ 323 seq_t seq; 324 struct vm_domain_policy vt_lcl; 325 326 for (;;) { 327 seq = seq_read(&vt->seq); 328 vt_lcl = *vt; 329 if (seq_consistent(&vt->seq, seq)) { 330 _vm_domain_iterator_set_policy(vi, &vt_lcl); 331 return; 332 } 333 cpu_spinwait(); 334 } 335} 336 337/* 338 * Return the next VM domain to use. 339 * 340 * Returns 0 w/ domain set to the next domain to use, or 341 * -1 to indicate no more domains are available. 342 */ 343int 344vm_domain_iterator_run(struct vm_domain_iterator *vi, int *domain) 345{ 346 347 /* General catch-all */ 348 if (vi->n <= 0) 349 return (-1); 350 351#ifdef VM_NUMA_ALLOC 352 switch (vi->policy) { 353 case VM_POLICY_FIXED_DOMAIN: 354 case VM_POLICY_FIRST_TOUCH: 355 *domain = vi->domain; 356 vi->n--; 357 break; 358 case VM_POLICY_FIXED_DOMAIN_ROUND_ROBIN: 359 case VM_POLICY_FIRST_TOUCH_ROUND_ROBIN: 360 /* 361 * XXX TODO: skip over the rr'ed domain 362 * if it equals the one we started with. 363 */ 364 if (vi->n == vm_ndomains) 365 *domain = vi->domain; 366 else 367 *domain = vm_domain_rr_selectdomain(vi->domain); 368 vi->n--; 369 break; 370 case VM_POLICY_ROUND_ROBIN: 371 default: 372 *domain = vm_domain_rr_selectdomain(-1); 373 vi->n--; 374 break; 375 } 376#else 377 *domain = 0; 378 vi->n--; 379#endif 380 381 return (0); 382} 383 384/* 385 * Returns 1 if the iteration is done, or 0 if it has not. 386 387 * This can only be called after at least one loop through 388 * the iterator. Ie, it's designed to be used as a tail 389 * check of a loop, not the head check of a loop. 390 */ 391int 392vm_domain_iterator_isdone(struct vm_domain_iterator *vi) 393{ 394 395 return (vi->n <= 0); 396} 397 398int 399vm_domain_iterator_cleanup(struct vm_domain_iterator *vi) 400{ 401 402 return (0); 403} 404