kern_cpuset.c revision 176730
1176730Sjeff/*- 2176730Sjeff * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 3176730Sjeff * All rights reserved. 4176730Sjeff * 5176730Sjeff * Redistribution and use in source and binary forms, with or without 6176730Sjeff * modification, are permitted provided that the following conditions 7176730Sjeff * are met: 8176730Sjeff * 1. Redistributions of source code must retain the above copyright 9176730Sjeff * notice unmodified, this list of conditions, and the following 10176730Sjeff * disclaimer. 11176730Sjeff * 2. Redistributions in binary form must reproduce the above copyright 12176730Sjeff * notice, this list of conditions and the following disclaimer in the 13176730Sjeff * documentation and/or other materials provided with the distribution. 14176730Sjeff * 15176730Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16176730Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17176730Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18176730Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19176730Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20176730Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21176730Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22176730Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23176730Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24176730Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25176730Sjeff * 26176730Sjeff */ 27176730Sjeff 28176730Sjeff#include <sys/cdefs.h> 29176730Sjeff__FBSDID("$FreeBSD: head/sys/kern/kern_cpuset.c 176730 2008-03-02 07:39:22Z jeff $"); 30176730Sjeff 31176730Sjeff#include <sys/param.h> 32176730Sjeff#include <sys/systm.h> 33176730Sjeff#include <sys/sysproto.h> 34176730Sjeff#include <sys/kernel.h> 35176730Sjeff#include <sys/lock.h> 36176730Sjeff#include <sys/malloc.h> 37176730Sjeff#include <sys/mutex.h> 38176730Sjeff#include <sys/priv.h> 39176730Sjeff#include <sys/proc.h> 40176730Sjeff#include <sys/refcount.h> 41176730Sjeff#include <sys/sched.h> 42176730Sjeff#include <sys/smp.h> 43176730Sjeff#include <sys/syscallsubr.h> 44176730Sjeff#include <sys/cpuset.h> 45176730Sjeff#include <sys/sx.h> 46176730Sjeff#include <sys/refcount.h> 47176730Sjeff#include <sys/queue.h> 48176730Sjeff#include <sys/limits.h> 49176730Sjeff 50176730Sjeff#include <vm/uma.h> 51176730Sjeff 52176730Sjeff/* 53176730Sjeff * cpusets provide a mechanism for creating and manipulating sets of 54176730Sjeff * processors for the purpose of constraining the scheduling of threads to 55176730Sjeff * specific processors. 56176730Sjeff * 57176730Sjeff * Each process belongs to an identified set, by default this is set 1. Each 58176730Sjeff * thread may further restrict the cpus it may run on to a subset of this 59176730Sjeff * named set. This creates an anonymous set which other threads and processes 60176730Sjeff * may not join by number. 61176730Sjeff * 62176730Sjeff * The named set is referred to herein as the 'base' set to avoid ambiguity. 63176730Sjeff * This set is usually a child of a 'root' set while the anonymous set may 64176730Sjeff * simply be referred to as a mask. In the syscall api these are referred to 65176730Sjeff * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 66176730Sjeff * 67176730Sjeff * Threads inherit their set from their creator whether it be anonymous or 68176730Sjeff * not. This means that anonymous sets are immutable because they may be 69176730Sjeff * shared. To modify an anonymous set a new set is created with the desired 70176730Sjeff * mask and the same parent as the existing anonymous set. This gives the 71176730Sjeff * illusion of each thread having a private mask.A 72176730Sjeff * 73176730Sjeff * Via the syscall apis a user may ask to retrieve or modify the root, base, 74176730Sjeff * or mask that is discovered via a pid, tid, or setid. Modifying a set 75176730Sjeff * modifies all numbered and anonymous child sets to comply with the new mask. 76176730Sjeff * Modifying a pid or tid's mask applies only to that tid but must still 77176730Sjeff * exist within the assigned parent set. 78176730Sjeff * 79176730Sjeff * A thread may not be assigned to a a group seperate from other threads in 80176730Sjeff * the process. This is to remove ambiguity when the setid is queried with 81176730Sjeff * a pid argument. There is no other technical limitation. 82176730Sjeff * 83176730Sjeff * This somewhat complex arrangement is intended to make it easy for 84176730Sjeff * applications to query available processors and bind their threads to 85176730Sjeff * specific processors while also allowing administrators to dynamically 86176730Sjeff * reprovision by changing sets which apply to groups of processes. 87176730Sjeff * 88176730Sjeff * A simple application should not concern itself with sets at all and 89176730Sjeff * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 90176730Sjeff * meaning 'curthread'. It may query availble cpus for that tid with a 91176730Sjeff * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 92176730Sjeff */ 93176730Sjeffstatic uma_zone_t cpuset_zone; 94176730Sjeffstatic struct mtx cpuset_lock; 95176730Sjeffstatic struct setlist cpuset_ids; 96176730Sjeffstruct cpuset *cpuset_zero; 97176730Sjeffstatic struct unrhdr *cpuset_unr; 98176730Sjeff 99176730Sjeff/* 100176730Sjeff * Acquire a reference to a cpuset, all pointers must be tracked with refs. 101176730Sjeff */ 102176730Sjeffstruct cpuset * 103176730Sjeffcpuset_ref(struct cpuset *set) 104176730Sjeff{ 105176730Sjeff 106176730Sjeff refcount_acquire(&set->cs_ref); 107176730Sjeff return (set); 108176730Sjeff} 109176730Sjeff 110176730Sjeff/* 111176730Sjeff * Release a reference in a context where it is safe to allocte. 112176730Sjeff */ 113176730Sjeffvoid 114176730Sjeffcpuset_rel(struct cpuset *set) 115176730Sjeff{ 116176730Sjeff cpusetid_t id; 117176730Sjeff 118176730Sjeff if (refcount_release(&set->cs_ref) == 0) 119176730Sjeff return; 120176730Sjeff mtx_lock_spin(&cpuset_lock); 121176730Sjeff LIST_REMOVE(set, cs_siblings); 122176730Sjeff id = set->cs_id; 123176730Sjeff if (id != CPUSET_INVALID) 124176730Sjeff LIST_REMOVE(set, cs_link); 125176730Sjeff mtx_unlock_spin(&cpuset_lock); 126176730Sjeff cpuset_rel(set->cs_parent); 127176730Sjeff uma_zfree(cpuset_zone, set); 128176730Sjeff if (id != CPUSET_INVALID) 129176730Sjeff free_unr(cpuset_unr, id); 130176730Sjeff} 131176730Sjeff 132176730Sjeff/* 133176730Sjeff * Deferred release must be used when in a context that is not safe to 134176730Sjeff * allocate/free. This places any unreferenced sets on the list 'head'. 135176730Sjeff */ 136176730Sjeffstatic void 137176730Sjeffcpuset_rel_defer(struct setlist *head, struct cpuset *set) 138176730Sjeff{ 139176730Sjeff 140176730Sjeff if (refcount_release(&set->cs_ref) == 0) 141176730Sjeff return; 142176730Sjeff mtx_lock_spin(&cpuset_lock); 143176730Sjeff LIST_REMOVE(set, cs_siblings); 144176730Sjeff if (set->cs_id != CPUSET_INVALID) 145176730Sjeff LIST_REMOVE(set, cs_link); 146176730Sjeff LIST_INSERT_HEAD(head, set, cs_link); 147176730Sjeff mtx_unlock_spin(&cpuset_lock); 148176730Sjeff} 149176730Sjeff 150176730Sjeff/* 151176730Sjeff * Complete a deferred release. Removes the set from the list provided to 152176730Sjeff * cpuset_rel_defer. 153176730Sjeff */ 154176730Sjeffstatic void 155176730Sjeffcpuset_rel_complete(struct cpuset *set) 156176730Sjeff{ 157176730Sjeff LIST_REMOVE(set, cs_link); 158176730Sjeff cpuset_rel(set->cs_parent); 159176730Sjeff uma_zfree(cpuset_zone, set); 160176730Sjeff} 161176730Sjeff 162176730Sjeff/* 163176730Sjeff * Find a set based on an id. Returns it with a ref. 164176730Sjeff */ 165176730Sjeffstatic struct cpuset * 166176730Sjeffcpuset_lookup(cpusetid_t setid) 167176730Sjeff{ 168176730Sjeff struct cpuset *set; 169176730Sjeff 170176730Sjeff if (setid == CPUSET_INVALID) 171176730Sjeff return (NULL); 172176730Sjeff mtx_lock_spin(&cpuset_lock); 173176730Sjeff LIST_FOREACH(set, &cpuset_ids, cs_link) 174176730Sjeff if (set->cs_id == setid) 175176730Sjeff break; 176176730Sjeff if (set) 177176730Sjeff cpuset_ref(set); 178176730Sjeff mtx_unlock_spin(&cpuset_lock); 179176730Sjeff return (set); 180176730Sjeff} 181176730Sjeff 182176730Sjeff/* 183176730Sjeff * Create a set in the space provided in 'set' with the provided parameters. 184176730Sjeff * The set is returned with a single ref. May return EDEADLK if the set 185176730Sjeff * will have no valid cpu based on restrictions from the parent. 186176730Sjeff */ 187176730Sjeffstatic int 188176730Sjeff_cpuset_create(struct cpuset *set, struct cpuset *parent, cpuset_t *mask, 189176730Sjeff cpusetid_t id) 190176730Sjeff{ 191176730Sjeff int error; 192176730Sjeff 193176730Sjeff error = 0; 194176730Sjeff CPU_COPY(mask, &set->cs_mask); 195176730Sjeff LIST_INIT(&set->cs_children); 196176730Sjeff refcount_init(&set->cs_ref, 1); 197176730Sjeff set->cs_flags = 0; 198176730Sjeff mtx_lock_spin(&cpuset_lock); 199176730Sjeff CPU_AND(mask, &parent->cs_mask); 200176730Sjeff if (!CPU_EMPTY(mask)) { 201176730Sjeff set->cs_id = id; 202176730Sjeff set->cs_parent = cpuset_ref(parent); 203176730Sjeff LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 204176730Sjeff if (set->cs_id != CPUSET_INVALID) 205176730Sjeff LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 206176730Sjeff } else 207176730Sjeff error = EDEADLK; 208176730Sjeff mtx_unlock_spin(&cpuset_lock); 209176730Sjeff 210176730Sjeff return (error); 211176730Sjeff} 212176730Sjeff 213176730Sjeff/* 214176730Sjeff * Create a new non-anonymous set with the requested parent and mask. May 215176730Sjeff * return failures if the mask is invalid or a new number can not be 216176730Sjeff * allocated. 217176730Sjeff */ 218176730Sjeffstatic int 219176730Sjeffcpuset_create(struct cpuset **setp, struct cpuset *parent, cpuset_t *mask) 220176730Sjeff{ 221176730Sjeff struct cpuset *set; 222176730Sjeff cpusetid_t id; 223176730Sjeff int error; 224176730Sjeff 225176730Sjeff id = alloc_unr(cpuset_unr); 226176730Sjeff if (id == -1) 227176730Sjeff return (ENFILE); 228176730Sjeff *setp = set = uma_zalloc(cpuset_zone, M_WAITOK); 229176730Sjeff error = _cpuset_create(set, parent, mask, id); 230176730Sjeff if (error == 0) 231176730Sjeff return (0); 232176730Sjeff free_unr(cpuset_unr, id); 233176730Sjeff uma_zfree(cpuset_zone, set); 234176730Sjeff 235176730Sjeff return (error); 236176730Sjeff} 237176730Sjeff 238176730Sjeff/* 239176730Sjeff * Recursively check for errors that would occur from applying mask to 240176730Sjeff * the tree of sets starting at 'set'. Checks for sets that would become 241176730Sjeff * empty as well as RDONLY flags. 242176730Sjeff */ 243176730Sjeffstatic int 244176730Sjeffcpuset_testupdate(struct cpuset *set, cpuset_t *mask) 245176730Sjeff{ 246176730Sjeff struct cpuset *nset; 247176730Sjeff cpuset_t newmask; 248176730Sjeff int error; 249176730Sjeff 250176730Sjeff mtx_assert(&cpuset_lock, MA_OWNED); 251176730Sjeff if (set->cs_flags & CPU_SET_RDONLY) 252176730Sjeff return (EPERM); 253176730Sjeff error = 0; 254176730Sjeff CPU_COPY(&set->cs_mask, &newmask); 255176730Sjeff CPU_AND(&newmask, mask); 256176730Sjeff if (CPU_EMPTY(&newmask)) 257176730Sjeff return (EDEADLK); 258176730Sjeff LIST_FOREACH(nset, &set->cs_children, cs_siblings) 259176730Sjeff if ((error = cpuset_testupdate(nset, &newmask)) != 0) 260176730Sjeff break; 261176730Sjeff return (error); 262176730Sjeff} 263176730Sjeff 264176730Sjeff/* 265176730Sjeff * Applies the mask 'mask' without checking for empty sets or permissions. 266176730Sjeff */ 267176730Sjeffstatic void 268176730Sjeffcpuset_update(struct cpuset *set, cpuset_t *mask) 269176730Sjeff{ 270176730Sjeff struct cpuset *nset; 271176730Sjeff 272176730Sjeff mtx_assert(&cpuset_lock, MA_OWNED); 273176730Sjeff CPU_AND(&set->cs_mask, mask); 274176730Sjeff LIST_FOREACH(nset, &set->cs_children, cs_siblings) 275176730Sjeff cpuset_update(nset, &set->cs_mask); 276176730Sjeff 277176730Sjeff return; 278176730Sjeff} 279176730Sjeff 280176730Sjeff/* 281176730Sjeff * Modify the set 'set' to use a copy of the mask provided. Apply this new 282176730Sjeff * mask to restrict all children in the tree. Checks for validity before 283176730Sjeff * applying the changes. 284176730Sjeff */ 285176730Sjeffstatic int 286176730Sjeffcpuset_modify(struct cpuset *set, cpuset_t *mask) 287176730Sjeff{ 288176730Sjeff int error; 289176730Sjeff 290176730Sjeff error = suser(curthread); 291176730Sjeff if (error) 292176730Sjeff return (error); 293176730Sjeff mtx_lock_spin(&cpuset_lock); 294176730Sjeff error = cpuset_testupdate(set, mask); 295176730Sjeff if (error) 296176730Sjeff goto out; 297176730Sjeff cpuset_update(set, mask); 298176730Sjeff CPU_COPY(mask, &set->cs_mask); 299176730Sjeffout: 300176730Sjeff mtx_unlock_spin(&cpuset_lock); 301176730Sjeff 302176730Sjeff return (error); 303176730Sjeff} 304176730Sjeff 305176730Sjeff/* 306176730Sjeff * Walks up the tree from 'set' to find the root. Returns the root 307176730Sjeff * referenced. 308176730Sjeff */ 309176730Sjeffstatic struct cpuset * 310176730Sjeffcpuset_root(struct cpuset *set) 311176730Sjeff{ 312176730Sjeff 313176730Sjeff mtx_lock_spin(&cpuset_lock); 314176730Sjeff for (; set->cs_parent != NULL; set = set->cs_parent) 315176730Sjeff if (set->cs_flags & CPU_SET_ROOT) 316176730Sjeff break; 317176730Sjeff cpuset_ref(set); 318176730Sjeff mtx_unlock_spin(&cpuset_lock); 319176730Sjeff 320176730Sjeff return (set); 321176730Sjeff} 322176730Sjeff 323176730Sjeff/* 324176730Sjeff * Find the first non-anonymous set starting from 'set'. Returns this set 325176730Sjeff * referenced. May return the passed in set with an extra ref if it is 326176730Sjeff * not anonymous. 327176730Sjeff */ 328176730Sjeffstatic struct cpuset * 329176730Sjeffcpuset_base(struct cpuset *set) 330176730Sjeff{ 331176730Sjeff 332176730Sjeff mtx_lock_spin(&cpuset_lock); 333176730Sjeff if (set->cs_id == CPUSET_INVALID) 334176730Sjeff set = set->cs_parent; 335176730Sjeff cpuset_ref(set); 336176730Sjeff mtx_unlock_spin(&cpuset_lock); 337176730Sjeff 338176730Sjeff return (set); 339176730Sjeff} 340176730Sjeff 341176730Sjeff/* 342176730Sjeff * Resolve the 'which' parameter of several cpuset apis. 343176730Sjeff * 344176730Sjeff * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 345176730Sjeff * checks for permission via p_cansched(). 346176730Sjeff * 347176730Sjeff * For WHICH_SET returns a valid set with a new reference. 348176730Sjeff * 349176730Sjeff * -1 may be supplied for any argument to mean the current proc/thread or 350176730Sjeff * the base set of the current thread. May fail with ESRCH/EPERM. 351176730Sjeff */ 352176730Sjeffstatic int 353176730Sjeffcpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 354176730Sjeff struct cpuset **setp) 355176730Sjeff{ 356176730Sjeff struct cpuset *set; 357176730Sjeff struct thread *td; 358176730Sjeff struct proc *p; 359176730Sjeff int error; 360176730Sjeff 361176730Sjeff *pp = p = NULL; 362176730Sjeff *tdp = td = NULL; 363176730Sjeff *setp = set = NULL; 364176730Sjeff switch (which) { 365176730Sjeff case CPU_WHICH_PID: 366176730Sjeff if (id == -1) { 367176730Sjeff PROC_LOCK(curproc); 368176730Sjeff p = curproc; 369176730Sjeff break; 370176730Sjeff } 371176730Sjeff if ((p = pfind(id)) == NULL) 372176730Sjeff return (ESRCH); 373176730Sjeff break; 374176730Sjeff case CPU_WHICH_TID: 375176730Sjeff if (id == -1) { 376176730Sjeff PROC_LOCK(curproc); 377176730Sjeff p = curproc; 378176730Sjeff td = curthread; 379176730Sjeff break; 380176730Sjeff } 381176730Sjeff sx_slock(&allproc_lock); 382176730Sjeff FOREACH_PROC_IN_SYSTEM(p) { 383176730Sjeff PROC_LOCK(p); 384176730Sjeff PROC_SLOCK(p); 385176730Sjeff FOREACH_THREAD_IN_PROC(p, td) 386176730Sjeff if (td->td_tid == id) 387176730Sjeff break; 388176730Sjeff PROC_SUNLOCK(p); 389176730Sjeff if (td != NULL) 390176730Sjeff break; 391176730Sjeff PROC_UNLOCK(p); 392176730Sjeff } 393176730Sjeff sx_sunlock(&allproc_lock); 394176730Sjeff if (td == NULL) 395176730Sjeff return (ESRCH); 396176730Sjeff break; 397176730Sjeff case CPU_WHICH_CPUSET: 398176730Sjeff if (id == -1) { 399176730Sjeff thread_lock(curthread); 400176730Sjeff set = cpuset_base(curthread->td_cpuset); 401176730Sjeff thread_unlock(curthread); 402176730Sjeff } else 403176730Sjeff set = cpuset_lookup(id); 404176730Sjeff if (set) { 405176730Sjeff *setp = set; 406176730Sjeff return (0); 407176730Sjeff } 408176730Sjeff return (ESRCH); 409176730Sjeff default: 410176730Sjeff return (EINVAL); 411176730Sjeff } 412176730Sjeff error = p_cansched(curthread, p); 413176730Sjeff if (error) { 414176730Sjeff PROC_UNLOCK(p); 415176730Sjeff return (error); 416176730Sjeff } 417176730Sjeff if (td == NULL) 418176730Sjeff td = FIRST_THREAD_IN_PROC(p); 419176730Sjeff *pp = p; 420176730Sjeff *tdp = td; 421176730Sjeff return (0); 422176730Sjeff} 423176730Sjeff 424176730Sjeff/* 425176730Sjeff * Create an anonymous set with the provided mask in the space provided by 426176730Sjeff * 'fset'. If the passed in set is anonymous we use its parent otherwise 427176730Sjeff * the new set is a child of 'set'. 428176730Sjeff */ 429176730Sjeffstatic int 430176730Sjeffcpuset_shadow(struct cpuset *set, struct cpuset *fset, cpuset_t *mask) 431176730Sjeff{ 432176730Sjeff struct cpuset *parent; 433176730Sjeff 434176730Sjeff if (set->cs_id == CPUSET_INVALID) 435176730Sjeff parent = set->cs_parent; 436176730Sjeff else 437176730Sjeff parent = set; 438176730Sjeff return (_cpuset_create(fset, parent, mask, CPUSET_INVALID)); 439176730Sjeff} 440176730Sjeff 441176730Sjeff/* 442176730Sjeff * Handle two cases for replacing the base set or mask of an entire process. 443176730Sjeff * 444176730Sjeff * 1) Set is non-null and mask is null. This reparents all anonymous sets 445176730Sjeff * to the provided set and replaces all non-anonymous td_cpusets with the 446176730Sjeff * provided set. 447176730Sjeff * 2) Mask is non-null and set is null. This replaces or creates anonymous 448176730Sjeff * sets for every thread with the existing base as a parent. 449176730Sjeff * 450176730Sjeff * This is overly complicated because we can't allocate while holding a 451176730Sjeff * spinlock and spinlocks must be held while changing and examining thread 452176730Sjeff * state. 453176730Sjeff */ 454176730Sjeffstatic int 455176730Sjeffcpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask) 456176730Sjeff{ 457176730Sjeff struct setlist freelist; 458176730Sjeff struct setlist droplist; 459176730Sjeff struct cpuset *nset; 460176730Sjeff struct thread *td; 461176730Sjeff struct proc *p; 462176730Sjeff int threads; 463176730Sjeff int nfree; 464176730Sjeff int error; 465176730Sjeff /* 466176730Sjeff * The algorithm requires two passes due to locking considerations. 467176730Sjeff * 468176730Sjeff * 1) Lookup the process and acquire the locks in the required order. 469176730Sjeff * 2) If enough cpusets have not been allocated release the locks and 470176730Sjeff * allocate them. Loop. 471176730Sjeff */ 472176730Sjeff LIST_INIT(&freelist); 473176730Sjeff LIST_INIT(&droplist); 474176730Sjeff nfree = 0; 475176730Sjeff for (;;) { 476176730Sjeff error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 477176730Sjeff if (error) 478176730Sjeff goto out; 479176730Sjeff PROC_SLOCK(p); 480176730Sjeff if (nfree >= p->p_numthreads) 481176730Sjeff break; 482176730Sjeff threads = p->p_numthreads; 483176730Sjeff PROC_SUNLOCK(p); 484176730Sjeff PROC_UNLOCK(p); 485176730Sjeff for (; nfree < threads; nfree++) { 486176730Sjeff nset = uma_zalloc(cpuset_zone, M_WAITOK); 487176730Sjeff LIST_INSERT_HEAD(&freelist, nset, cs_link); 488176730Sjeff } 489176730Sjeff } 490176730Sjeff PROC_LOCK_ASSERT(p, MA_OWNED); 491176730Sjeff PROC_SLOCK_ASSERT(p, MA_OWNED); 492176730Sjeff /* 493176730Sjeff * Now that the appropriate locks are held and we have enough cpusets, 494176730Sjeff * replace each thread's cpuset while using deferred release. We 495176730Sjeff * must do this because the PROC_SLOCK has to be held while traversing 496176730Sjeff * the thread list and this limits the type of operations allowed. 497176730Sjeff */ 498176730Sjeff error = 0; 499176730Sjeff FOREACH_THREAD_IN_PROC(p, td) { 500176730Sjeff struct cpuset *tdset; 501176730Sjeff thread_lock(td); 502176730Sjeff /* 503176730Sjeff * If we presently have an anonymous set or are applying a 504176730Sjeff * mask we must create an anonymous shadow set. That is 505176730Sjeff * either parented to our existing base or the supplied set. 506176730Sjeff * 507176730Sjeff * If we have a base set with no anonymous shadow we simply 508176730Sjeff * replace it outright. 509176730Sjeff */ 510176730Sjeff tdset = td->td_cpuset; 511176730Sjeff if (tdset->cs_id == CPUSET_INVALID || mask) { 512176730Sjeff nset = LIST_FIRST(&freelist); 513176730Sjeff LIST_REMOVE(nset, cs_link); 514176730Sjeff if (mask) 515176730Sjeff error = cpuset_shadow(tdset, nset, mask); 516176730Sjeff else 517176730Sjeff error = _cpuset_create(nset, set, 518176730Sjeff &tdset->cs_mask, CPUSET_INVALID); 519176730Sjeff if (error) { 520176730Sjeff LIST_INSERT_HEAD(&freelist, nset, cs_link); 521176730Sjeff thread_unlock(td); 522176730Sjeff break; 523176730Sjeff } 524176730Sjeff } else 525176730Sjeff nset = cpuset_ref(set); 526176730Sjeff cpuset_rel_defer(&droplist, tdset); 527176730Sjeff td->td_cpuset = nset; 528176730Sjeff sched_affinity(td); 529176730Sjeff thread_unlock(td); 530176730Sjeff } 531176730Sjeff PROC_SUNLOCK(p); 532176730Sjeff PROC_UNLOCK(p); 533176730Sjeffout: 534176730Sjeff while ((nset = LIST_FIRST(&droplist)) != NULL) 535176730Sjeff cpuset_rel_complete(nset); 536176730Sjeff while ((nset = LIST_FIRST(&freelist)) != NULL) { 537176730Sjeff LIST_REMOVE(nset, cs_link); 538176730Sjeff uma_zfree(cpuset_zone, nset); 539176730Sjeff } 540176730Sjeff return (error); 541176730Sjeff} 542176730Sjeff 543176730Sjeff/* 544176730Sjeff * Apply an anonymous mask to a single thread. 545176730Sjeff */ 546176730Sjeffstatic int 547176730Sjeffcpuset_setthread(lwpid_t id, cpuset_t *mask) 548176730Sjeff{ 549176730Sjeff struct cpuset *nset; 550176730Sjeff struct cpuset *set; 551176730Sjeff struct thread *td; 552176730Sjeff struct proc *p; 553176730Sjeff int error; 554176730Sjeff 555176730Sjeff nset = uma_zalloc(cpuset_zone, M_WAITOK); 556176730Sjeff error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &nset); 557176730Sjeff if (error) 558176730Sjeff goto out; 559176730Sjeff thread_lock(td); 560176730Sjeff set = td->td_cpuset; 561176730Sjeff error = cpuset_shadow(set, nset, mask); 562176730Sjeff if (error == 0) { 563176730Sjeff cpuset_rel(td->td_cpuset); 564176730Sjeff td->td_cpuset = nset; 565176730Sjeff sched_affinity(td); 566176730Sjeff nset = NULL; 567176730Sjeff } 568176730Sjeff thread_unlock(td); 569176730Sjeff PROC_UNLOCK(p); 570176730Sjeffout: 571176730Sjeff if (nset) 572176730Sjeff uma_zfree(cpuset_zone, nset); 573176730Sjeff return (error); 574176730Sjeff} 575176730Sjeff 576176730Sjeff/* 577176730Sjeff * Creates the cpuset for thread0. We make two sets: 578176730Sjeff * 579176730Sjeff * 0 - The root set which should represent all valid processors in the 580176730Sjeff * system. It is initially created with a mask of all processors 581176730Sjeff * because we don't know what processors are valid until cpuset_init() 582176730Sjeff * runs. This set is immutable. 583176730Sjeff * 1 - The default set which all processes are a member of until changed. 584176730Sjeff * This allows an administrator to move all threads off of given cpus to 585176730Sjeff * dedicate them to high priority tasks or save power etc. 586176730Sjeff */ 587176730Sjeffstruct cpuset * 588176730Sjeffcpuset_thread0(void) 589176730Sjeff{ 590176730Sjeff struct cpuset *set; 591176730Sjeff int error; 592176730Sjeff 593176730Sjeff cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 594176730Sjeff NULL, NULL, UMA_ALIGN_PTR, 0); 595176730Sjeff mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 596176730Sjeff /* 597176730Sjeff * Create the root system set for the whole machine. Doesn't use 598176730Sjeff * cpuset_create() due to NULL parent. 599176730Sjeff */ 600176730Sjeff set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 601176730Sjeff set->cs_mask.__bits[0] = -1; 602176730Sjeff LIST_INIT(&set->cs_children); 603176730Sjeff LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 604176730Sjeff set->cs_ref = 1; 605176730Sjeff set->cs_flags = CPU_SET_ROOT; 606176730Sjeff cpuset_zero = set; 607176730Sjeff /* 608176730Sjeff * Now derive a default, modifiable set from that to give out. 609176730Sjeff */ 610176730Sjeff set = uma_zalloc(cpuset_zone, M_WAITOK); 611176730Sjeff error = _cpuset_create(set, cpuset_zero, &cpuset_zero->cs_mask, 1); 612176730Sjeff KASSERT(error == 0, ("Error creating default set: %d\n", error)); 613176730Sjeff /* 614176730Sjeff * Initialize the unit allocator. 0 and 1 are allocated above. 615176730Sjeff */ 616176730Sjeff cpuset_unr = new_unrhdr(2, INT_MAX, NULL); 617176730Sjeff 618176730Sjeff return (set); 619176730Sjeff} 620176730Sjeff 621176730Sjeff/* 622176730Sjeff * This is called once the final set of system cpus is known. Modifies 623176730Sjeff * the root set and all children and mark the root readonly. 624176730Sjeff */ 625176730Sjeffstatic void 626176730Sjeffcpuset_init(void *arg) 627176730Sjeff{ 628176730Sjeff cpuset_t mask; 629176730Sjeff 630176730Sjeff CPU_ZERO(&mask); 631176730Sjeff#ifdef SMP 632176730Sjeff mask.__bits[0] = all_cpus; 633176730Sjeff#else 634176730Sjeff mask.__bits[0] = 1; 635176730Sjeff#endif 636176730Sjeff if (cpuset_modify(cpuset_zero, &mask)) 637176730Sjeff panic("Can't set initial cpuset mask.\n"); 638176730Sjeff cpuset_zero->cs_flags |= CPU_SET_RDONLY; 639176730Sjeff} 640176730SjeffSYSINIT(cpuset, SI_SUB_SMP, SI_ORDER_ANY, cpuset_init, NULL); 641176730Sjeff 642176730Sjeff#ifndef _SYS_SYSPROTO_H_ 643176730Sjeffstruct cpuset_args { 644176730Sjeff cpusetid_t *setid; 645176730Sjeff}; 646176730Sjeff#endif 647176730Sjeffint 648176730Sjeffcpuset(struct thread *td, struct cpuset_args *uap) 649176730Sjeff{ 650176730Sjeff struct cpuset *root; 651176730Sjeff struct cpuset *set; 652176730Sjeff int error; 653176730Sjeff 654176730Sjeff thread_lock(td); 655176730Sjeff root = cpuset_root(td->td_cpuset); 656176730Sjeff thread_unlock(td); 657176730Sjeff error = cpuset_create(&set, root, &root->cs_mask); 658176730Sjeff cpuset_rel(root); 659176730Sjeff if (error) 660176730Sjeff return (error); 661176730Sjeff error = cpuset_setproc(-1, set, NULL); 662176730Sjeff if (error == 0) 663176730Sjeff error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 664176730Sjeff cpuset_rel(set); 665176730Sjeff return (error); 666176730Sjeff} 667176730Sjeff 668176730Sjeff#ifndef _SYS_SYSPROTO_H_ 669176730Sjeffstruct cpuset_setid_args { 670176730Sjeff cpuwhich_t which; 671176730Sjeff id_t id; 672176730Sjeff cpusetid_t setid; 673176730Sjeff}; 674176730Sjeff#endif 675176730Sjeffint 676176730Sjeffcpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 677176730Sjeff{ 678176730Sjeff struct cpuset *set; 679176730Sjeff int error; 680176730Sjeff 681176730Sjeff /* 682176730Sjeff * Presently we only support per-process sets. 683176730Sjeff */ 684176730Sjeff if (uap->which != CPU_WHICH_PID) 685176730Sjeff return (EINVAL); 686176730Sjeff set = cpuset_lookup(uap->setid); 687176730Sjeff if (set == NULL) 688176730Sjeff return (ESRCH); 689176730Sjeff error = cpuset_setproc(uap->id, set, NULL); 690176730Sjeff cpuset_rel(set); 691176730Sjeff return (error); 692176730Sjeff} 693176730Sjeff 694176730Sjeff#ifndef _SYS_SYSPROTO_H_ 695176730Sjeffstruct cpuset_getid_args { 696176730Sjeff cpulevel_t level; 697176730Sjeff cpuwhich_t which; 698176730Sjeff id_t id; 699176730Sjeff cpusetid_t *setid; 700176730Sjeff#endif 701176730Sjeffint 702176730Sjeffcpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 703176730Sjeff{ 704176730Sjeff struct cpuset *nset; 705176730Sjeff struct cpuset *set; 706176730Sjeff struct thread *ttd; 707176730Sjeff struct proc *p; 708176730Sjeff cpusetid_t id; 709176730Sjeff int error; 710176730Sjeff 711176730Sjeff if (uap->level == CPU_LEVEL_WHICH && uap->which != CPU_WHICH_CPUSET) 712176730Sjeff return (EINVAL); 713176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 714176730Sjeff if (error) 715176730Sjeff return (error); 716176730Sjeff switch (uap->which) { 717176730Sjeff case CPU_WHICH_TID: 718176730Sjeff case CPU_WHICH_PID: 719176730Sjeff thread_lock(ttd); 720176730Sjeff set = cpuset_base(ttd->td_cpuset); 721176730Sjeff thread_unlock(ttd); 722176730Sjeff PROC_UNLOCK(p); 723176730Sjeff break; 724176730Sjeff case CPU_WHICH_CPUSET: 725176730Sjeff break; 726176730Sjeff } 727176730Sjeff switch (uap->level) { 728176730Sjeff case CPU_LEVEL_ROOT: 729176730Sjeff nset = cpuset_root(set); 730176730Sjeff cpuset_rel(set); 731176730Sjeff set = nset; 732176730Sjeff break; 733176730Sjeff case CPU_LEVEL_CPUSET: 734176730Sjeff break; 735176730Sjeff case CPU_LEVEL_WHICH: 736176730Sjeff break; 737176730Sjeff } 738176730Sjeff id = set->cs_id; 739176730Sjeff cpuset_rel(set); 740176730Sjeff if (error == 0) 741176730Sjeff error = copyout(&id, uap->setid, sizeof(id)); 742176730Sjeff 743176730Sjeff return (error); 744176730Sjeff} 745176730Sjeff 746176730Sjeff#ifndef _SYS_SYSPROTO_H_ 747176730Sjeffstruct cpuset_getaffinity_args { 748176730Sjeff cpulevel_t level; 749176730Sjeff cpuwhich_t which; 750176730Sjeff int id; 751176730Sjeff int cpusetsize; 752176730Sjeff long *mask; 753176730Sjeff}; 754176730Sjeff#endif 755176730Sjeffint 756176730Sjeffcpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 757176730Sjeff{ 758176730Sjeff struct thread *ttd; 759176730Sjeff struct cpuset *nset; 760176730Sjeff struct cpuset *set; 761176730Sjeff struct proc *p; 762176730Sjeff cpuset_t *mask; 763176730Sjeff int error; 764176730Sjeff int size; 765176730Sjeff 766176730Sjeff if (uap->cpusetsize < CPU_SETSIZE || uap->cpusetsize > CPU_MAXSIZE) 767176730Sjeff return (ERANGE); 768176730Sjeff size = uap->cpusetsize / NBBY; 769176730Sjeff mask = malloc(size, M_TEMP, M_WAITOK | M_ZERO); 770176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 771176730Sjeff if (error) 772176730Sjeff goto out; 773176730Sjeff error = 0; 774176730Sjeff switch (uap->level) { 775176730Sjeff case CPU_LEVEL_ROOT: 776176730Sjeff case CPU_LEVEL_CPUSET: 777176730Sjeff switch (uap->which) { 778176730Sjeff case CPU_WHICH_TID: 779176730Sjeff case CPU_WHICH_PID: 780176730Sjeff thread_lock(ttd); 781176730Sjeff set = cpuset_ref(ttd->td_cpuset); 782176730Sjeff thread_unlock(ttd); 783176730Sjeff break; 784176730Sjeff case CPU_WHICH_CPUSET: 785176730Sjeff break; 786176730Sjeff } 787176730Sjeff if (uap->level == CPU_LEVEL_ROOT) 788176730Sjeff nset = cpuset_root(set); 789176730Sjeff else 790176730Sjeff nset = cpuset_base(set); 791176730Sjeff CPU_COPY(&nset->cs_mask, mask); 792176730Sjeff cpuset_rel(nset); 793176730Sjeff break; 794176730Sjeff case CPU_LEVEL_WHICH: 795176730Sjeff switch (uap->which) { 796176730Sjeff case CPU_WHICH_TID: 797176730Sjeff thread_lock(ttd); 798176730Sjeff CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 799176730Sjeff thread_unlock(ttd); 800176730Sjeff break; 801176730Sjeff case CPU_WHICH_PID: 802176730Sjeff PROC_SLOCK(p); 803176730Sjeff FOREACH_THREAD_IN_PROC(p, ttd) { 804176730Sjeff thread_lock(ttd); 805176730Sjeff CPU_OR(mask, &ttd->td_cpuset->cs_mask); 806176730Sjeff thread_unlock(ttd); 807176730Sjeff } 808176730Sjeff PROC_SUNLOCK(p); 809176730Sjeff break; 810176730Sjeff case CPU_WHICH_CPUSET: 811176730Sjeff CPU_COPY(&set->cs_mask, mask); 812176730Sjeff break; 813176730Sjeff } 814176730Sjeff break; 815176730Sjeff default: 816176730Sjeff error = EINVAL; 817176730Sjeff break; 818176730Sjeff } 819176730Sjeff if (set) 820176730Sjeff cpuset_rel(set); 821176730Sjeff if (p) 822176730Sjeff PROC_UNLOCK(p); 823176730Sjeff if (error == 0) 824176730Sjeff error = copyout(mask, uap->mask, size); 825176730Sjeffout: 826176730Sjeff free(mask, M_TEMP); 827176730Sjeff return (error); 828176730Sjeff} 829176730Sjeff 830176730Sjeff#ifndef _SYS_SYSPROTO_H_ 831176730Sjeffstruct cpuset_setaffinity_args { 832176730Sjeff cpulevel_t level; 833176730Sjeff cpuwhich_t which; 834176730Sjeff int id; 835176730Sjeff int cpusetsize; 836176730Sjeff long * mask; 837176730Sjeff}; 838176730Sjeff#endif 839176730Sjeffint 840176730Sjeffcpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 841176730Sjeff{ 842176730Sjeff struct cpuset *nset; 843176730Sjeff struct cpuset *set; 844176730Sjeff struct thread *ttd; 845176730Sjeff struct proc *p; 846176730Sjeff cpuset_t *mask; 847176730Sjeff int error; 848176730Sjeff 849176730Sjeff if (uap->cpusetsize < CPU_SETSIZE || uap->cpusetsize > CPU_MAXSIZE) 850176730Sjeff return (ERANGE); 851176730Sjeff mask = malloc(uap->cpusetsize / NBBY, M_TEMP, M_WAITOK | M_ZERO); 852176730Sjeff error = copyin(uap->mask, mask, uap->cpusetsize / NBBY); 853176730Sjeff if (error) 854176730Sjeff goto out; 855176730Sjeff switch (uap->level) { 856176730Sjeff case CPU_LEVEL_ROOT: 857176730Sjeff case CPU_LEVEL_CPUSET: 858176730Sjeff error = cpuset_which(uap->which, uap->id, &p, &ttd, &set); 859176730Sjeff if (error) 860176730Sjeff break; 861176730Sjeff switch (uap->which) { 862176730Sjeff case CPU_WHICH_TID: 863176730Sjeff case CPU_WHICH_PID: 864176730Sjeff thread_lock(ttd); 865176730Sjeff set = cpuset_ref(ttd->td_cpuset); 866176730Sjeff thread_unlock(ttd); 867176730Sjeff break; 868176730Sjeff case CPU_WHICH_CPUSET: 869176730Sjeff break; 870176730Sjeff } 871176730Sjeff if (uap->level == CPU_LEVEL_ROOT) 872176730Sjeff nset = cpuset_root(set); 873176730Sjeff else 874176730Sjeff nset = cpuset_base(set); 875176730Sjeff error = cpuset_modify(nset, mask); 876176730Sjeff cpuset_rel(nset); 877176730Sjeff cpuset_rel(set); 878176730Sjeff break; 879176730Sjeff case CPU_LEVEL_WHICH: 880176730Sjeff switch (uap->which) { 881176730Sjeff case CPU_WHICH_TID: 882176730Sjeff error = cpuset_setthread(uap->id, mask); 883176730Sjeff break; 884176730Sjeff case CPU_WHICH_PID: 885176730Sjeff error = cpuset_setproc(uap->id, NULL, mask); 886176730Sjeff break; 887176730Sjeff case CPU_WHICH_CPUSET: 888176730Sjeff error = cpuset_which(CPU_WHICH_CPUSET, uap->id, &p, 889176730Sjeff &ttd, &set); 890176730Sjeff if (error == 0) { 891176730Sjeff error = cpuset_modify(set, mask); 892176730Sjeff cpuset_rel(set); 893176730Sjeff } 894176730Sjeff break; 895176730Sjeff default: 896176730Sjeff error = EINVAL; 897176730Sjeff break; 898176730Sjeff } 899176730Sjeff break; 900176730Sjeff default: 901176730Sjeff error = EINVAL; 902176730Sjeff break; 903176730Sjeff } 904176730Sjeffout: 905176730Sjeff free(mask, M_TEMP); 906176730Sjeff return (error); 907176730Sjeff} 908