kern_cpuset.c revision 177904
1176730Sjeff/*- 2176730Sjeff * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 3176730Sjeff * All rights reserved. 4177904Sjeff * 5177904Sjeff * Copyright (c) 2008 Nokia Corporation 6177904Sjeff * All rights reserved. 7176730Sjeff * 8176730Sjeff * Redistribution and use in source and binary forms, with or without 9176730Sjeff * modification, are permitted provided that the following conditions 10176730Sjeff * are met: 11176730Sjeff * 1. Redistributions of source code must retain the above copyright 12176730Sjeff * notice unmodified, this list of conditions, and the following 13176730Sjeff * disclaimer. 14176730Sjeff * 2. Redistributions in binary form must reproduce the above copyright 15176730Sjeff * notice, this list of conditions and the following disclaimer in the 16176730Sjeff * documentation and/or other materials provided with the distribution. 17176730Sjeff * 18176730Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19176730Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20176730Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21176730Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22176730Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23176730Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24176730Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25176730Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26176730Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27176730Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28176730Sjeff * 29176730Sjeff */ 30176730Sjeff 31176730Sjeff#include <sys/cdefs.h> 32176730Sjeff__FBSDID("$FreeBSD: head/sys/kern/kern_cpuset.c 177904 2008-04-04 01:22:04Z jeff $"); 33176730Sjeff 34176730Sjeff#include <sys/param.h> 35176730Sjeff#include <sys/systm.h> 36176730Sjeff#include <sys/sysproto.h> 37176730Sjeff#include <sys/kernel.h> 38176730Sjeff#include <sys/lock.h> 39176730Sjeff#include <sys/malloc.h> 40176730Sjeff#include <sys/mutex.h> 41176730Sjeff#include <sys/priv.h> 42176730Sjeff#include <sys/proc.h> 43176730Sjeff#include <sys/refcount.h> 44176730Sjeff#include <sys/sched.h> 45176730Sjeff#include <sys/smp.h> 46176730Sjeff#include <sys/syscallsubr.h> 47176730Sjeff#include <sys/cpuset.h> 48176730Sjeff#include <sys/sx.h> 49176730Sjeff#include <sys/refcount.h> 50176730Sjeff#include <sys/queue.h> 51176730Sjeff#include <sys/limits.h> 52177738Sjeff#include <sys/bus.h> 53177738Sjeff#include <sys/interrupt.h> 54176730Sjeff 55176730Sjeff#include <vm/uma.h> 56176730Sjeff 57176730Sjeff/* 58176730Sjeff * cpusets provide a mechanism for creating and manipulating sets of 59176730Sjeff * processors for the purpose of constraining the scheduling of threads to 60176730Sjeff * specific processors. 61176730Sjeff * 62176730Sjeff * Each process belongs to an identified set, by default this is set 1. Each 63176730Sjeff * thread may further restrict the cpus it may run on to a subset of this 64176730Sjeff * named set. This creates an anonymous set which other threads and processes 65176730Sjeff * may not join by number. 66176730Sjeff * 67176730Sjeff * The named set is referred to herein as the 'base' set to avoid ambiguity. 68176730Sjeff * This set is usually a child of a 'root' set while the anonymous set may 69176730Sjeff * simply be referred to as a mask. In the syscall api these are referred to 70176730Sjeff * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 71176730Sjeff * 72176730Sjeff * Threads inherit their set from their creator whether it be anonymous or 73176730Sjeff * not. This means that anonymous sets are immutable because they may be 74176730Sjeff * shared. To modify an anonymous set a new set is created with the desired 75176730Sjeff * mask and the same parent as the existing anonymous set. This gives the 76176730Sjeff * illusion of each thread having a private mask.A 77176730Sjeff * 78176730Sjeff * Via the syscall apis a user may ask to retrieve or modify the root, base, 79176730Sjeff * or mask that is discovered via a pid, tid, or setid. Modifying a set 80176730Sjeff * modifies all numbered and anonymous child sets to comply with the new mask. 81176730Sjeff * Modifying a pid or tid's mask applies only to that tid but must still 82176730Sjeff * exist within the assigned parent set. 83176730Sjeff * 84176730Sjeff * A thread may not be assigned to a a group seperate from other threads in 85176730Sjeff * the process. This is to remove ambiguity when the setid is queried with 86176730Sjeff * a pid argument. There is no other technical limitation. 87176730Sjeff * 88176730Sjeff * This somewhat complex arrangement is intended to make it easy for 89176730Sjeff * applications to query available processors and bind their threads to 90176730Sjeff * specific processors while also allowing administrators to dynamically 91176730Sjeff * reprovision by changing sets which apply to groups of processes. 92176730Sjeff * 93176730Sjeff * A simple application should not concern itself with sets at all and 94176730Sjeff * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 95176730Sjeff * meaning 'curthread'. It may query availble cpus for that tid with a 96176730Sjeff * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 97176730Sjeff */ 98176730Sjeffstatic uma_zone_t cpuset_zone; 99176730Sjeffstatic struct mtx cpuset_lock; 100176730Sjeffstatic struct setlist cpuset_ids; 101176730Sjeffstatic struct unrhdr *cpuset_unr; 102177738Sjeffstatic struct cpuset *cpuset_zero; 103176730Sjeff 104177738Sjeffcpuset_t *cpuset_root; 105177738Sjeff 106176730Sjeff/* 107176730Sjeff * Acquire a reference to a cpuset, all pointers must be tracked with refs. 108176730Sjeff */ 109176730Sjeffstruct cpuset * 110176730Sjeffcpuset_ref(struct cpuset *set) 111176730Sjeff{ 112176730Sjeff 113176730Sjeff refcount_acquire(&set->cs_ref); 114176730Sjeff return (set); 115176730Sjeff} 116176730Sjeff 117176730Sjeff/* 118176730Sjeff * Release a reference in a context where it is safe to allocte. 119176730Sjeff */ 120176730Sjeffvoid 121176730Sjeffcpuset_rel(struct cpuset *set) 122176730Sjeff{ 123176730Sjeff cpusetid_t id; 124176730Sjeff 125176730Sjeff if (refcount_release(&set->cs_ref) == 0) 126176730Sjeff return; 127176730Sjeff mtx_lock_spin(&cpuset_lock); 128176730Sjeff LIST_REMOVE(set, cs_siblings); 129176730Sjeff id = set->cs_id; 130176730Sjeff if (id != CPUSET_INVALID) 131176730Sjeff LIST_REMOVE(set, cs_link); 132176730Sjeff mtx_unlock_spin(&cpuset_lock); 133176730Sjeff cpuset_rel(set->cs_parent); 134176730Sjeff uma_zfree(cpuset_zone, set); 135176730Sjeff if (id != CPUSET_INVALID) 136176730Sjeff free_unr(cpuset_unr, id); 137176730Sjeff} 138176730Sjeff 139176730Sjeff/* 140176730Sjeff * Deferred release must be used when in a context that is not safe to 141176730Sjeff * allocate/free. This places any unreferenced sets on the list 'head'. 142176730Sjeff */ 143176730Sjeffstatic void 144176730Sjeffcpuset_rel_defer(struct setlist *head, struct cpuset *set) 145176730Sjeff{ 146176730Sjeff 147176730Sjeff if (refcount_release(&set->cs_ref) == 0) 148176730Sjeff return; 149176730Sjeff mtx_lock_spin(&cpuset_lock); 150176730Sjeff LIST_REMOVE(set, cs_siblings); 151176730Sjeff if (set->cs_id != CPUSET_INVALID) 152176730Sjeff LIST_REMOVE(set, cs_link); 153176730Sjeff LIST_INSERT_HEAD(head, set, cs_link); 154176730Sjeff mtx_unlock_spin(&cpuset_lock); 155176730Sjeff} 156176730Sjeff 157176730Sjeff/* 158176730Sjeff * Complete a deferred release. Removes the set from the list provided to 159176730Sjeff * cpuset_rel_defer. 160176730Sjeff */ 161176730Sjeffstatic void 162176730Sjeffcpuset_rel_complete(struct cpuset *set) 163176730Sjeff{ 164176730Sjeff LIST_REMOVE(set, cs_link); 165176730Sjeff cpuset_rel(set->cs_parent); 166176730Sjeff uma_zfree(cpuset_zone, set); 167176730Sjeff} 168176730Sjeff 169176730Sjeff/* 170176730Sjeff * Find a set based on an id. Returns it with a ref. 171176730Sjeff */ 172176730Sjeffstatic struct cpuset * 173176730Sjeffcpuset_lookup(cpusetid_t setid) 174176730Sjeff{ 175176730Sjeff struct cpuset *set; 176176730Sjeff 177176730Sjeff if (setid == CPUSET_INVALID) 178176730Sjeff return (NULL); 179176730Sjeff mtx_lock_spin(&cpuset_lock); 180176730Sjeff LIST_FOREACH(set, &cpuset_ids, cs_link) 181176730Sjeff if (set->cs_id == setid) 182176730Sjeff break; 183176730Sjeff if (set) 184176730Sjeff cpuset_ref(set); 185176730Sjeff mtx_unlock_spin(&cpuset_lock); 186176730Sjeff return (set); 187176730Sjeff} 188176730Sjeff 189176730Sjeff/* 190176730Sjeff * Create a set in the space provided in 'set' with the provided parameters. 191176730Sjeff * The set is returned with a single ref. May return EDEADLK if the set 192176730Sjeff * will have no valid cpu based on restrictions from the parent. 193176730Sjeff */ 194176730Sjeffstatic int 195176730Sjeff_cpuset_create(struct cpuset *set, struct cpuset *parent, cpuset_t *mask, 196176730Sjeff cpusetid_t id) 197176730Sjeff{ 198176730Sjeff 199176811Sjeff if (!CPU_OVERLAP(&parent->cs_mask, mask)) 200176811Sjeff return (EDEADLK); 201176730Sjeff CPU_COPY(mask, &set->cs_mask); 202176730Sjeff LIST_INIT(&set->cs_children); 203176730Sjeff refcount_init(&set->cs_ref, 1); 204176730Sjeff set->cs_flags = 0; 205176730Sjeff mtx_lock_spin(&cpuset_lock); 206176730Sjeff CPU_AND(mask, &parent->cs_mask); 207176811Sjeff set->cs_id = id; 208176811Sjeff set->cs_parent = cpuset_ref(parent); 209176811Sjeff LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 210176811Sjeff if (set->cs_id != CPUSET_INVALID) 211176811Sjeff LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 212176730Sjeff mtx_unlock_spin(&cpuset_lock); 213176730Sjeff 214176811Sjeff return (0); 215176730Sjeff} 216176730Sjeff 217176730Sjeff/* 218176730Sjeff * Create a new non-anonymous set with the requested parent and mask. May 219176730Sjeff * return failures if the mask is invalid or a new number can not be 220176730Sjeff * allocated. 221176730Sjeff */ 222176730Sjeffstatic int 223176730Sjeffcpuset_create(struct cpuset **setp, struct cpuset *parent, cpuset_t *mask) 224176730Sjeff{ 225176730Sjeff struct cpuset *set; 226176730Sjeff cpusetid_t id; 227176730Sjeff int error; 228176730Sjeff 229176730Sjeff id = alloc_unr(cpuset_unr); 230176730Sjeff if (id == -1) 231176730Sjeff return (ENFILE); 232176730Sjeff *setp = set = uma_zalloc(cpuset_zone, M_WAITOK); 233176730Sjeff error = _cpuset_create(set, parent, mask, id); 234176730Sjeff if (error == 0) 235176730Sjeff return (0); 236176730Sjeff free_unr(cpuset_unr, id); 237176730Sjeff uma_zfree(cpuset_zone, set); 238176730Sjeff 239176730Sjeff return (error); 240176730Sjeff} 241176730Sjeff 242176730Sjeff/* 243176730Sjeff * Recursively check for errors that would occur from applying mask to 244176730Sjeff * the tree of sets starting at 'set'. Checks for sets that would become 245176730Sjeff * empty as well as RDONLY flags. 246176730Sjeff */ 247176730Sjeffstatic int 248176730Sjeffcpuset_testupdate(struct cpuset *set, cpuset_t *mask) 249176730Sjeff{ 250176730Sjeff struct cpuset *nset; 251176730Sjeff cpuset_t newmask; 252176730Sjeff int error; 253176730Sjeff 254176730Sjeff mtx_assert(&cpuset_lock, MA_OWNED); 255176730Sjeff if (set->cs_flags & CPU_SET_RDONLY) 256176730Sjeff return (EPERM); 257176811Sjeff if (!CPU_OVERLAP(&set->cs_mask, mask)) 258176811Sjeff return (EDEADLK); 259176730Sjeff CPU_COPY(&set->cs_mask, &newmask); 260176730Sjeff CPU_AND(&newmask, mask); 261176811Sjeff error = 0; 262176730Sjeff LIST_FOREACH(nset, &set->cs_children, cs_siblings) 263176730Sjeff if ((error = cpuset_testupdate(nset, &newmask)) != 0) 264176730Sjeff break; 265176730Sjeff return (error); 266176730Sjeff} 267176730Sjeff 268176730Sjeff/* 269176730Sjeff * Applies the mask 'mask' without checking for empty sets or permissions. 270176730Sjeff */ 271176730Sjeffstatic void 272176730Sjeffcpuset_update(struct cpuset *set, cpuset_t *mask) 273176730Sjeff{ 274176730Sjeff struct cpuset *nset; 275176730Sjeff 276176730Sjeff mtx_assert(&cpuset_lock, MA_OWNED); 277176730Sjeff CPU_AND(&set->cs_mask, mask); 278176730Sjeff LIST_FOREACH(nset, &set->cs_children, cs_siblings) 279176730Sjeff cpuset_update(nset, &set->cs_mask); 280176730Sjeff 281176730Sjeff return; 282176730Sjeff} 283176730Sjeff 284176730Sjeff/* 285176730Sjeff * Modify the set 'set' to use a copy of the mask provided. Apply this new 286176730Sjeff * mask to restrict all children in the tree. Checks for validity before 287176730Sjeff * applying the changes. 288176730Sjeff */ 289176730Sjeffstatic int 290176730Sjeffcpuset_modify(struct cpuset *set, cpuset_t *mask) 291176730Sjeff{ 292176811Sjeff struct cpuset *root; 293176730Sjeff int error; 294176730Sjeff 295176730Sjeff error = suser(curthread); 296176730Sjeff if (error) 297176730Sjeff return (error); 298176811Sjeff /* 299176811Sjeff * Verify that we have access to this set of 300176811Sjeff * cpus. 301176811Sjeff */ 302176811Sjeff root = set->cs_parent; 303176811Sjeff if (root && !CPU_SUBSET(&root->cs_mask, mask)) 304176811Sjeff return (EINVAL); 305176730Sjeff mtx_lock_spin(&cpuset_lock); 306176730Sjeff error = cpuset_testupdate(set, mask); 307176730Sjeff if (error) 308176730Sjeff goto out; 309176730Sjeff cpuset_update(set, mask); 310176730Sjeff CPU_COPY(mask, &set->cs_mask); 311176730Sjeffout: 312176730Sjeff mtx_unlock_spin(&cpuset_lock); 313176730Sjeff 314176730Sjeff return (error); 315176730Sjeff} 316176730Sjeff 317176730Sjeff/* 318176730Sjeff * Walks up the tree from 'set' to find the root. Returns the root 319176730Sjeff * referenced. 320176730Sjeff */ 321176730Sjeffstatic struct cpuset * 322177738Sjeffcpuset_refroot(struct cpuset *set) 323176730Sjeff{ 324176730Sjeff 325176730Sjeff for (; set->cs_parent != NULL; set = set->cs_parent) 326176730Sjeff if (set->cs_flags & CPU_SET_ROOT) 327176730Sjeff break; 328176730Sjeff cpuset_ref(set); 329176730Sjeff 330176730Sjeff return (set); 331176730Sjeff} 332176730Sjeff 333176730Sjeff/* 334176730Sjeff * Find the first non-anonymous set starting from 'set'. Returns this set 335176730Sjeff * referenced. May return the passed in set with an extra ref if it is 336176730Sjeff * not anonymous. 337176730Sjeff */ 338176730Sjeffstatic struct cpuset * 339177738Sjeffcpuset_refbase(struct cpuset *set) 340176730Sjeff{ 341176730Sjeff 342176730Sjeff if (set->cs_id == CPUSET_INVALID) 343176730Sjeff set = set->cs_parent; 344176730Sjeff cpuset_ref(set); 345176730Sjeff 346176730Sjeff return (set); 347176730Sjeff} 348176730Sjeff 349176730Sjeff/* 350176730Sjeff * Resolve the 'which' parameter of several cpuset apis. 351176730Sjeff * 352176730Sjeff * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 353176730Sjeff * checks for permission via p_cansched(). 354176730Sjeff * 355176730Sjeff * For WHICH_SET returns a valid set with a new reference. 356176730Sjeff * 357176730Sjeff * -1 may be supplied for any argument to mean the current proc/thread or 358176730Sjeff * the base set of the current thread. May fail with ESRCH/EPERM. 359176730Sjeff */ 360176730Sjeffstatic int 361176730Sjeffcpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 362176730Sjeff struct cpuset **setp) 363176730Sjeff{ 364176730Sjeff struct cpuset *set; 365176730Sjeff struct thread *td; 366176730Sjeff struct proc *p; 367176730Sjeff int error; 368176730Sjeff 369176730Sjeff *pp = p = NULL; 370176730Sjeff *tdp = td = NULL; 371176730Sjeff *setp = set = NULL; 372176730Sjeff switch (which) { 373176730Sjeff case CPU_WHICH_PID: 374176730Sjeff if (id == -1) { 375176730Sjeff PROC_LOCK(curproc); 376176730Sjeff p = curproc; 377176730Sjeff break; 378176730Sjeff } 379176730Sjeff if ((p = pfind(id)) == NULL) 380176730Sjeff return (ESRCH); 381176730Sjeff break; 382176730Sjeff case CPU_WHICH_TID: 383176730Sjeff if (id == -1) { 384176730Sjeff PROC_LOCK(curproc); 385176730Sjeff p = curproc; 386176730Sjeff td = curthread; 387176730Sjeff break; 388176730Sjeff } 389176730Sjeff sx_slock(&allproc_lock); 390176730Sjeff FOREACH_PROC_IN_SYSTEM(p) { 391176730Sjeff PROC_LOCK(p); 392176730Sjeff FOREACH_THREAD_IN_PROC(p, td) 393176730Sjeff if (td->td_tid == id) 394176730Sjeff break; 395176730Sjeff if (td != NULL) 396176730Sjeff break; 397176730Sjeff PROC_UNLOCK(p); 398176730Sjeff } 399176730Sjeff sx_sunlock(&allproc_lock); 400176730Sjeff if (td == NULL) 401176730Sjeff return (ESRCH); 402176730Sjeff break; 403176730Sjeff case CPU_WHICH_CPUSET: 404176730Sjeff if (id == -1) { 405176730Sjeff thread_lock(curthread); 406177738Sjeff set = cpuset_refbase(curthread->td_cpuset); 407176730Sjeff thread_unlock(curthread); 408176730Sjeff } else 409176730Sjeff set = cpuset_lookup(id); 410176730Sjeff if (set) { 411176730Sjeff *setp = set; 412176730Sjeff return (0); 413176730Sjeff } 414176730Sjeff return (ESRCH); 415176730Sjeff default: 416176730Sjeff return (EINVAL); 417176730Sjeff } 418176730Sjeff error = p_cansched(curthread, p); 419176730Sjeff if (error) { 420176730Sjeff PROC_UNLOCK(p); 421176730Sjeff return (error); 422176730Sjeff } 423176730Sjeff if (td == NULL) 424176730Sjeff td = FIRST_THREAD_IN_PROC(p); 425176730Sjeff *pp = p; 426176730Sjeff *tdp = td; 427176730Sjeff return (0); 428176730Sjeff} 429176730Sjeff 430176730Sjeff/* 431176730Sjeff * Create an anonymous set with the provided mask in the space provided by 432176730Sjeff * 'fset'. If the passed in set is anonymous we use its parent otherwise 433176730Sjeff * the new set is a child of 'set'. 434176730Sjeff */ 435176730Sjeffstatic int 436176730Sjeffcpuset_shadow(struct cpuset *set, struct cpuset *fset, cpuset_t *mask) 437176730Sjeff{ 438176730Sjeff struct cpuset *parent; 439176730Sjeff 440176730Sjeff if (set->cs_id == CPUSET_INVALID) 441176730Sjeff parent = set->cs_parent; 442176730Sjeff else 443176730Sjeff parent = set; 444176811Sjeff if (!CPU_SUBSET(&parent->cs_mask, mask)) 445177738Sjeff return (EDEADLK); 446176730Sjeff return (_cpuset_create(fset, parent, mask, CPUSET_INVALID)); 447176730Sjeff} 448176730Sjeff 449176730Sjeff/* 450176730Sjeff * Handle two cases for replacing the base set or mask of an entire process. 451176730Sjeff * 452176730Sjeff * 1) Set is non-null and mask is null. This reparents all anonymous sets 453176730Sjeff * to the provided set and replaces all non-anonymous td_cpusets with the 454176730Sjeff * provided set. 455176730Sjeff * 2) Mask is non-null and set is null. This replaces or creates anonymous 456176730Sjeff * sets for every thread with the existing base as a parent. 457176730Sjeff * 458176730Sjeff * This is overly complicated because we can't allocate while holding a 459176730Sjeff * spinlock and spinlocks must be held while changing and examining thread 460176730Sjeff * state. 461176730Sjeff */ 462176730Sjeffstatic int 463176730Sjeffcpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask) 464176730Sjeff{ 465176730Sjeff struct setlist freelist; 466176730Sjeff struct setlist droplist; 467176811Sjeff struct cpuset *tdset; 468176730Sjeff struct cpuset *nset; 469176730Sjeff struct thread *td; 470176730Sjeff struct proc *p; 471176730Sjeff int threads; 472176730Sjeff int nfree; 473176730Sjeff int error; 474176730Sjeff /* 475176730Sjeff * The algorithm requires two passes due to locking considerations. 476176730Sjeff * 477176730Sjeff * 1) Lookup the process and acquire the locks in the required order. 478176730Sjeff * 2) If enough cpusets have not been allocated release the locks and 479176730Sjeff * allocate them. Loop. 480176730Sjeff */ 481176730Sjeff LIST_INIT(&freelist); 482176730Sjeff LIST_INIT(&droplist); 483176730Sjeff nfree = 0; 484176730Sjeff for (;;) { 485176730Sjeff error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 486176730Sjeff if (error) 487176730Sjeff goto out; 488176730Sjeff if (nfree >= p->p_numthreads) 489176730Sjeff break; 490176730Sjeff threads = p->p_numthreads; 491176730Sjeff PROC_UNLOCK(p); 492176730Sjeff for (; nfree < threads; nfree++) { 493176730Sjeff nset = uma_zalloc(cpuset_zone, M_WAITOK); 494176730Sjeff LIST_INSERT_HEAD(&freelist, nset, cs_link); 495176730Sjeff } 496176730Sjeff } 497176730Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 498176730Sjeff /* 499176730Sjeff * Now that the appropriate locks are held and we have enough cpusets, 500176811Sjeff * make sure the operation will succeed before applying changes. The 501176811Sjeff * proc lock prevents td_cpuset from changing between calls. 502176811Sjeff */ 503176811Sjeff error = 0; 504176811Sjeff FOREACH_THREAD_IN_PROC(p, td) { 505176811Sjeff thread_lock(td); 506176811Sjeff tdset = td->td_cpuset; 507176811Sjeff /* 508176811Sjeff * Verify that a new mask doesn't specify cpus outside of 509176811Sjeff * the set the thread is a member of. 510176811Sjeff */ 511176811Sjeff if (mask) { 512176811Sjeff if (tdset->cs_id == CPUSET_INVALID) 513176811Sjeff tdset = tdset->cs_parent; 514176811Sjeff if (!CPU_SUBSET(&tdset->cs_mask, mask)) 515177738Sjeff error = EDEADLK; 516176811Sjeff /* 517176811Sjeff * Verify that a new set won't leave an existing thread 518176811Sjeff * mask without a cpu to run on. It can, however, restrict 519176811Sjeff * the set. 520176811Sjeff */ 521176811Sjeff } else if (tdset->cs_id == CPUSET_INVALID) { 522176811Sjeff if (!CPU_OVERLAP(&set->cs_mask, &tdset->cs_mask)) 523177738Sjeff error = EDEADLK; 524176811Sjeff } 525176811Sjeff thread_unlock(td); 526176811Sjeff if (error) 527176811Sjeff goto unlock_out; 528176811Sjeff } 529176811Sjeff /* 530176811Sjeff * Replace each thread's cpuset while using deferred release. We 531177368Sjeff * must do this because the thread lock must be held while operating 532177368Sjeff * on the thread and this limits the type of operations allowed. 533176730Sjeff */ 534176730Sjeff FOREACH_THREAD_IN_PROC(p, td) { 535176730Sjeff thread_lock(td); 536176730Sjeff /* 537176730Sjeff * If we presently have an anonymous set or are applying a 538176730Sjeff * mask we must create an anonymous shadow set. That is 539176730Sjeff * either parented to our existing base or the supplied set. 540176730Sjeff * 541176730Sjeff * If we have a base set with no anonymous shadow we simply 542176730Sjeff * replace it outright. 543176730Sjeff */ 544176730Sjeff tdset = td->td_cpuset; 545176730Sjeff if (tdset->cs_id == CPUSET_INVALID || mask) { 546176730Sjeff nset = LIST_FIRST(&freelist); 547176730Sjeff LIST_REMOVE(nset, cs_link); 548176730Sjeff if (mask) 549176730Sjeff error = cpuset_shadow(tdset, nset, mask); 550176730Sjeff else 551176730Sjeff error = _cpuset_create(nset, set, 552176730Sjeff &tdset->cs_mask, CPUSET_INVALID); 553176730Sjeff if (error) { 554176730Sjeff LIST_INSERT_HEAD(&freelist, nset, cs_link); 555176730Sjeff thread_unlock(td); 556176730Sjeff break; 557176730Sjeff } 558176730Sjeff } else 559176730Sjeff nset = cpuset_ref(set); 560176730Sjeff cpuset_rel_defer(&droplist, tdset); 561176730Sjeff td->td_cpuset = nset; 562176730Sjeff sched_affinity(td); 563176730Sjeff thread_unlock(td); 564176730Sjeff } 565176811Sjeffunlock_out: 566176730Sjeff PROC_UNLOCK(p); 567176730Sjeffout: 568176730Sjeff while ((nset = LIST_FIRST(&droplist)) != NULL) 569176730Sjeff cpuset_rel_complete(nset); 570176730Sjeff while ((nset = LIST_FIRST(&freelist)) != NULL) { 571176730Sjeff LIST_REMOVE(nset, cs_link); 572176730Sjeff uma_zfree(cpuset_zone, nset); 573176730Sjeff } 574176730Sjeff return (error); 575176730Sjeff} 576176730Sjeff 577176730Sjeff/* 578176730Sjeff * Apply an anonymous mask to a single thread. 579176730Sjeff */ 580177738Sjeffint 581176730Sjeffcpuset_setthread(lwpid_t id, cpuset_t *mask) 582176730Sjeff{ 583176730Sjeff struct cpuset *nset; 584176730Sjeff struct cpuset *set; 585176730Sjeff struct thread *td; 586176730Sjeff struct proc *p; 587176730Sjeff int error; 588176730Sjeff 589176730Sjeff nset = uma_zalloc(cpuset_zone, M_WAITOK); 590176821Sjeff error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &set); 591176730Sjeff if (error) 592176730Sjeff goto out; 593177738Sjeff set = NULL; 594176730Sjeff thread_lock(td); 595177738Sjeff error = cpuset_shadow(td->td_cpuset, nset, mask); 596176730Sjeff if (error == 0) { 597177738Sjeff set = td->td_cpuset; 598176730Sjeff td->td_cpuset = nset; 599176730Sjeff sched_affinity(td); 600176730Sjeff nset = NULL; 601176730Sjeff } 602176730Sjeff thread_unlock(td); 603176730Sjeff PROC_UNLOCK(p); 604177738Sjeff if (set) 605177738Sjeff cpuset_rel(set); 606176730Sjeffout: 607176730Sjeff if (nset) 608176730Sjeff uma_zfree(cpuset_zone, nset); 609176730Sjeff return (error); 610176730Sjeff} 611176730Sjeff 612176730Sjeff/* 613176730Sjeff * Creates the cpuset for thread0. We make two sets: 614176730Sjeff * 615176730Sjeff * 0 - The root set which should represent all valid processors in the 616176730Sjeff * system. It is initially created with a mask of all processors 617176730Sjeff * because we don't know what processors are valid until cpuset_init() 618176730Sjeff * runs. This set is immutable. 619176730Sjeff * 1 - The default set which all processes are a member of until changed. 620176730Sjeff * This allows an administrator to move all threads off of given cpus to 621176730Sjeff * dedicate them to high priority tasks or save power etc. 622176730Sjeff */ 623176730Sjeffstruct cpuset * 624176730Sjeffcpuset_thread0(void) 625176730Sjeff{ 626176730Sjeff struct cpuset *set; 627176730Sjeff int error; 628176730Sjeff 629176730Sjeff cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 630176730Sjeff NULL, NULL, UMA_ALIGN_PTR, 0); 631176730Sjeff mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 632176730Sjeff /* 633176730Sjeff * Create the root system set for the whole machine. Doesn't use 634176730Sjeff * cpuset_create() due to NULL parent. 635176730Sjeff */ 636176730Sjeff set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 637176730Sjeff set->cs_mask.__bits[0] = -1; 638176730Sjeff LIST_INIT(&set->cs_children); 639176730Sjeff LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 640176730Sjeff set->cs_ref = 1; 641176730Sjeff set->cs_flags = CPU_SET_ROOT; 642176730Sjeff cpuset_zero = set; 643177738Sjeff cpuset_root = &set->cs_mask; 644176730Sjeff /* 645176730Sjeff * Now derive a default, modifiable set from that to give out. 646176730Sjeff */ 647176730Sjeff set = uma_zalloc(cpuset_zone, M_WAITOK); 648176730Sjeff error = _cpuset_create(set, cpuset_zero, &cpuset_zero->cs_mask, 1); 649176730Sjeff KASSERT(error == 0, ("Error creating default set: %d\n", error)); 650176730Sjeff /* 651176730Sjeff * Initialize the unit allocator. 0 and 1 are allocated above. 652176730Sjeff */ 653176730Sjeff cpuset_unr = new_unrhdr(2, INT_MAX, NULL); 654176730Sjeff 655176730Sjeff return (set); 656176730Sjeff} 657176730Sjeff 658176730Sjeff/* 659176730Sjeff * This is called once the final set of system cpus is known. Modifies 660176730Sjeff * the root set and all children and mark the root readonly. 661176730Sjeff */ 662176730Sjeffstatic void 663176730Sjeffcpuset_init(void *arg) 664176730Sjeff{ 665176730Sjeff cpuset_t mask; 666176730Sjeff 667176730Sjeff CPU_ZERO(&mask); 668176730Sjeff#ifdef SMP 669176730Sjeff mask.__bits[0] = all_cpus; 670176730Sjeff#else 671176730Sjeff mask.__bits[0] = 1; 672176730Sjeff#endif 673176730Sjeff if (cpuset_modify(cpuset_zero, &mask)) 674176730Sjeff panic("Can't set initial cpuset mask.\n"); 675176730Sjeff cpuset_zero->cs_flags |= CPU_SET_RDONLY; 676176730Sjeff} 677176730SjeffSYSINIT(cpuset, SI_SUB_SMP, SI_ORDER_ANY, cpuset_init, NULL); 678176730Sjeff 679176730Sjeff#ifndef _SYS_SYSPROTO_H_ 680176730Sjeffstruct cpuset_args { 681176730Sjeff cpusetid_t *setid; 682176730Sjeff}; 683176730Sjeff#endif 684176730Sjeffint 685176730Sjeffcpuset(struct thread *td, struct cpuset_args *uap) 686176730Sjeff{ 687176730Sjeff struct cpuset *root; 688176730Sjeff struct cpuset *set; 689176730Sjeff int error; 690176730Sjeff 691176730Sjeff thread_lock(td); 692177738Sjeff root = cpuset_refroot(td->td_cpuset); 693176730Sjeff thread_unlock(td); 694176730Sjeff error = cpuset_create(&set, root, &root->cs_mask); 695176730Sjeff cpuset_rel(root); 696176730Sjeff if (error) 697176730Sjeff return (error); 698177738Sjeff error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 699176730Sjeff if (error == 0) 700177738Sjeff error = cpuset_setproc(-1, set, NULL); 701176730Sjeff cpuset_rel(set); 702176730Sjeff return (error); 703176730Sjeff} 704176730Sjeff 705176730Sjeff#ifndef _SYS_SYSPROTO_H_ 706176730Sjeffstruct cpuset_setid_args { 707176730Sjeff cpuwhich_t which; 708176730Sjeff id_t id; 709176730Sjeff cpusetid_t setid; 710176730Sjeff}; 711176730Sjeff#endif 712176730Sjeffint 713176730Sjeffcpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 714176730Sjeff{ 715176730Sjeff struct cpuset *set; 716176730Sjeff int error; 717176730Sjeff 718176730Sjeff /* 719176730Sjeff * Presently we only support per-process sets. 720176730Sjeff */ 721176730Sjeff if (uap->which != CPU_WHICH_PID) 722176730Sjeff return (EINVAL); 723176730Sjeff set = cpuset_lookup(uap->setid); 724176730Sjeff if (set == NULL) 725176730Sjeff return (ESRCH); 726176730Sjeff error = cpuset_setproc(uap->id, set, NULL); 727176730Sjeff cpuset_rel(set); 728176730Sjeff return (error); 729176730Sjeff} 730176730Sjeff 731176730Sjeff#ifndef _SYS_SYSPROTO_H_ 732176730Sjeffstruct cpuset_getid_args { 733176730Sjeff cpulevel_t level; 734176730Sjeff cpuwhich_t which; 735176730Sjeff id_t id; 736176730Sjeff cpusetid_t *setid; 737176730Sjeff#endif 738176730Sjeffint 739176730Sjeffcpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 740176730Sjeff{ 741176730Sjeff struct cpuset *nset; 742176730Sjeff struct cpuset *set; 743176730Sjeff struct thread *ttd; 744176730Sjeff struct proc *p; 745176730Sjeff cpusetid_t id; 746176730Sjeff int error; 747176730Sjeff 748176730Sjeff if (uap->level == CPU_LEVEL_WHICH && uap->which != CPU_WHICH_CPUSET) 749176730Sjeff return (EINVAL); 750176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 751176730Sjeff if (error) 752176730Sjeff return (error); 753176730Sjeff switch (uap->which) { 754176730Sjeff case CPU_WHICH_TID: 755176730Sjeff case CPU_WHICH_PID: 756176730Sjeff thread_lock(ttd); 757177738Sjeff set = cpuset_refbase(ttd->td_cpuset); 758176730Sjeff thread_unlock(ttd); 759176730Sjeff PROC_UNLOCK(p); 760176730Sjeff break; 761176730Sjeff case CPU_WHICH_CPUSET: 762176730Sjeff break; 763176730Sjeff } 764176730Sjeff switch (uap->level) { 765176730Sjeff case CPU_LEVEL_ROOT: 766177738Sjeff nset = cpuset_refroot(set); 767176730Sjeff cpuset_rel(set); 768176730Sjeff set = nset; 769176730Sjeff break; 770176730Sjeff case CPU_LEVEL_CPUSET: 771176730Sjeff break; 772176730Sjeff case CPU_LEVEL_WHICH: 773176730Sjeff break; 774176730Sjeff } 775176730Sjeff id = set->cs_id; 776176730Sjeff cpuset_rel(set); 777176730Sjeff if (error == 0) 778176730Sjeff error = copyout(&id, uap->setid, sizeof(id)); 779176730Sjeff 780176730Sjeff return (error); 781176730Sjeff} 782176730Sjeff 783176730Sjeff#ifndef _SYS_SYSPROTO_H_ 784176730Sjeffstruct cpuset_getaffinity_args { 785177597Sru cpulevel_t level; 786177597Sru cpuwhich_t which; 787177597Sru id_t id; 788177597Sru size_t cpusetsize; 789177597Sru cpuset_t *mask; 790176730Sjeff}; 791176730Sjeff#endif 792176730Sjeffint 793176730Sjeffcpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 794176730Sjeff{ 795176730Sjeff struct thread *ttd; 796176730Sjeff struct cpuset *nset; 797176730Sjeff struct cpuset *set; 798176730Sjeff struct proc *p; 799176730Sjeff cpuset_t *mask; 800176730Sjeff int error; 801177597Sru size_t size; 802176730Sjeff 803176811Sjeff if (uap->cpusetsize < sizeof(cpuset_t) || 804176811Sjeff uap->cpusetsize * NBBY > CPU_MAXSIZE) 805176730Sjeff return (ERANGE); 806176811Sjeff size = uap->cpusetsize; 807176730Sjeff mask = malloc(size, M_TEMP, M_WAITOK | M_ZERO); 808176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 809176730Sjeff if (error) 810176730Sjeff goto out; 811176730Sjeff switch (uap->level) { 812176730Sjeff case CPU_LEVEL_ROOT: 813176730Sjeff case CPU_LEVEL_CPUSET: 814176730Sjeff switch (uap->which) { 815176730Sjeff case CPU_WHICH_TID: 816176730Sjeff case CPU_WHICH_PID: 817176730Sjeff thread_lock(ttd); 818176730Sjeff set = cpuset_ref(ttd->td_cpuset); 819176730Sjeff thread_unlock(ttd); 820176730Sjeff break; 821176730Sjeff case CPU_WHICH_CPUSET: 822176730Sjeff break; 823176730Sjeff } 824176730Sjeff if (uap->level == CPU_LEVEL_ROOT) 825177738Sjeff nset = cpuset_refroot(set); 826176730Sjeff else 827177738Sjeff nset = cpuset_refbase(set); 828176730Sjeff CPU_COPY(&nset->cs_mask, mask); 829176730Sjeff cpuset_rel(nset); 830176730Sjeff break; 831176730Sjeff case CPU_LEVEL_WHICH: 832176730Sjeff switch (uap->which) { 833176730Sjeff case CPU_WHICH_TID: 834176730Sjeff thread_lock(ttd); 835176730Sjeff CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 836176730Sjeff thread_unlock(ttd); 837176730Sjeff break; 838176730Sjeff case CPU_WHICH_PID: 839176730Sjeff FOREACH_THREAD_IN_PROC(p, ttd) { 840176730Sjeff thread_lock(ttd); 841176730Sjeff CPU_OR(mask, &ttd->td_cpuset->cs_mask); 842176730Sjeff thread_unlock(ttd); 843176730Sjeff } 844176730Sjeff break; 845176730Sjeff case CPU_WHICH_CPUSET: 846176730Sjeff CPU_COPY(&set->cs_mask, mask); 847176730Sjeff break; 848176730Sjeff } 849176730Sjeff break; 850176730Sjeff default: 851176730Sjeff error = EINVAL; 852176730Sjeff break; 853176730Sjeff } 854176730Sjeff if (set) 855176730Sjeff cpuset_rel(set); 856176730Sjeff if (p) 857176730Sjeff PROC_UNLOCK(p); 858176730Sjeff if (error == 0) 859176730Sjeff error = copyout(mask, uap->mask, size); 860176730Sjeffout: 861176730Sjeff free(mask, M_TEMP); 862176730Sjeff return (error); 863176730Sjeff} 864176730Sjeff 865176730Sjeff#ifndef _SYS_SYSPROTO_H_ 866176730Sjeffstruct cpuset_setaffinity_args { 867176730Sjeff cpulevel_t level; 868177597Sru cpuwhich_t which; 869177597Sru id_t id; 870177597Sru size_t cpusetsize; 871177597Sru const cpuset_t *mask; 872176730Sjeff}; 873176730Sjeff#endif 874176730Sjeffint 875176730Sjeffcpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 876176730Sjeff{ 877176730Sjeff struct cpuset *nset; 878176730Sjeff struct cpuset *set; 879176730Sjeff struct thread *ttd; 880176730Sjeff struct proc *p; 881176730Sjeff cpuset_t *mask; 882176730Sjeff int error; 883176730Sjeff 884176811Sjeff if (uap->cpusetsize < sizeof(cpuset_t) || 885176811Sjeff uap->cpusetsize * NBBY > CPU_MAXSIZE) 886176730Sjeff return (ERANGE); 887176811Sjeff mask = malloc(uap->cpusetsize, M_TEMP, M_WAITOK | M_ZERO); 888176811Sjeff error = copyin(uap->mask, mask, uap->cpusetsize); 889176730Sjeff if (error) 890176730Sjeff goto out; 891176811Sjeff /* 892176811Sjeff * Verify that no high bits are set. 893176811Sjeff */ 894176811Sjeff if (uap->cpusetsize > sizeof(cpuset_t)) { 895176811Sjeff char *end; 896176811Sjeff char *cp; 897176811Sjeff 898176811Sjeff end = cp = (char *)&mask->__bits; 899176811Sjeff end += uap->cpusetsize; 900176811Sjeff cp += sizeof(cpuset_t); 901176811Sjeff while (cp != end) 902176811Sjeff if (*cp++ != 0) { 903176811Sjeff error = EINVAL; 904176811Sjeff goto out; 905176811Sjeff } 906176811Sjeff 907176811Sjeff } 908176730Sjeff switch (uap->level) { 909176730Sjeff case CPU_LEVEL_ROOT: 910176730Sjeff case CPU_LEVEL_CPUSET: 911176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 912176730Sjeff if (error) 913176730Sjeff break; 914176730Sjeff switch (uap->which) { 915176730Sjeff case CPU_WHICH_TID: 916176730Sjeff case CPU_WHICH_PID: 917176730Sjeff thread_lock(ttd); 918176730Sjeff set = cpuset_ref(ttd->td_cpuset); 919176730Sjeff thread_unlock(ttd); 920176880Sjeff PROC_UNLOCK(p); 921176730Sjeff break; 922176730Sjeff case CPU_WHICH_CPUSET: 923176730Sjeff break; 924176730Sjeff } 925176730Sjeff if (uap->level == CPU_LEVEL_ROOT) 926177738Sjeff nset = cpuset_refroot(set); 927176730Sjeff else 928177738Sjeff nset = cpuset_refbase(set); 929176730Sjeff error = cpuset_modify(nset, mask); 930176730Sjeff cpuset_rel(nset); 931176730Sjeff cpuset_rel(set); 932176730Sjeff break; 933176730Sjeff case CPU_LEVEL_WHICH: 934176730Sjeff switch (uap->which) { 935176730Sjeff case CPU_WHICH_TID: 936176730Sjeff error = cpuset_setthread(uap->id, mask); 937176730Sjeff break; 938176730Sjeff case CPU_WHICH_PID: 939176730Sjeff error = cpuset_setproc(uap->id, NULL, mask); 940176730Sjeff break; 941176730Sjeff case CPU_WHICH_CPUSET: 942176730Sjeff error = cpuset_which(CPU_WHICH_CPUSET, uap->id, &p, 943176730Sjeff &ttd, &set); 944176730Sjeff if (error == 0) { 945176730Sjeff error = cpuset_modify(set, mask); 946176730Sjeff cpuset_rel(set); 947176730Sjeff } 948176730Sjeff break; 949176730Sjeff default: 950176730Sjeff error = EINVAL; 951176730Sjeff break; 952176730Sjeff } 953176730Sjeff break; 954176730Sjeff default: 955176730Sjeff error = EINVAL; 956176730Sjeff break; 957176730Sjeff } 958176730Sjeffout: 959176730Sjeff free(mask, M_TEMP); 960176730Sjeff return (error); 961176730Sjeff} 962