Lines Matching refs:cpuset
2 * kernel/cpuset.c
27 #include <linux/cpuset.h>
53 * There could be abnormal cpuset configurations for cpu or memory
84 [PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus.exclusive",
87 [PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive",
90 [PERR_CPUSEMPTY] = "cpuset.cpus is empty",
94 struct cpuset {
103 * cpuset.cpus and cpuset.mems, and won't be limited by the
107 * in the cpuset. They may be changed if the configured masks are
150 * - A new cpuset's old_mems_allowed is initialized when some
153 * cpuset.mems_allowed and have tasks' nodemask updated, and
161 * Tasks are being attached to this cpuset. Used to prevent
187 * number of SCHED_DEADLINE tasks attached to this cpuset, so that we
197 /* Handle for cpuset.cpus.partition */
246 static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
248 return css ? container_of(css, struct cpuset, css) : NULL;
251 /* Retrieve the cpuset for a task */
252 static inline struct cpuset *task_cs(struct task_struct *task)
257 static inline struct cpuset *parent_cs(struct cpuset *cs)
264 struct cpuset *cs = task_cs(p);
271 struct cpuset *cs = task_cs(p);
276 /* bits in struct cpuset flags field */
289 static inline bool is_cpuset_online(struct cpuset *cs)
294 static inline int is_cpu_exclusive(const struct cpuset *cs)
299 static inline int is_mem_exclusive(const struct cpuset *cs)
304 static inline int is_mem_hardwall(const struct cpuset *cs)
309 static inline int is_sched_load_balance(const struct cpuset *cs)
314 static inline int is_memory_migrate(const struct cpuset *cs)
319 static inline int is_spread_page(const struct cpuset *cs)
324 static inline int is_spread_slab(const struct cpuset *cs)
329 static inline int is_partition_valid(const struct cpuset *cs)
334 static inline int is_partition_invalid(const struct cpuset *cs)
342 static inline void make_partition_invalid(struct cpuset *cs)
351 static inline void notify_partition_change(struct cpuset *cs, int old_prs)
362 static struct cpuset top_cpuset = {
370 * cpuset_for_each_child - traverse online children of a cpuset
373 * @parent_cs: target cpuset to walk children of
383 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
386 * @root_cs: target cpuset to walk ancestor of
398 * There are two global locks guarding cpuset structures - cpuset_mutex and
400 * task's cpuset pointer. See "The task_lock() exception", at the end of this
401 * comment. The cpuset code uses only cpuset_mutex. Other kernel subsystems
402 * can use cpuset_lock()/cpuset_unlock() to prevent change to cpuset
410 * various checks on the cpuset structure first, knowing nothing will change.
418 * from one of the callbacks into the cpuset code from within
432 * Accessing a task's cpuset should be done in accordance with the
465 pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"
474 * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
488 * @excluded_child: a child cpuset to be excluded in task checking
492 * be non-NULL when this cpuset is going to become a partition itself.
494 static inline bool partition_is_populated(struct cpuset *cs,
495 struct cpuset *excluded_child)
498 struct cpuset *child;
523 * walk up the cpuset hierarchy until we find one that does have some
535 struct cpuset *cs;
547 * The top cpuset doesn't have any online cpu as a
550 * cpuset's effective_cpus is on its way to be
565 * up the cpuset hierarchy until we find one that does have some
566 * online mems. The top cpuset always has some mems online.
573 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
581 * update task's spread flag if cpuset's page/slab spread flag is set
586 static void cpuset_update_task_spread_flags(struct cpuset *cs,
604 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
606 * One cpuset is a subset of another if all its allowed CPUs and
611 static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
620 * alloc_cpumasks - allocate three cpumasks for cpuset
621 * @cs: the cpuset that have cpumasks to be allocated.
627 static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
669 * @cs: the cpuset that have cpumasks to be free.
672 static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
688 * alloc_trial_cpuset - allocate a trial cpuset
689 * @cs: the cpuset that the trial cpuset duplicates
691 static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
693 struct cpuset *trial;
712 * free_cpuset - free the cpuset
713 * @cs: the cpuset to be freed
715 static inline void free_cpuset(struct cpuset *cs)
721 static inline struct cpumask *fetch_xcpus(struct cpuset *cs)
733 static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2)
747 static int validate_change_legacy(struct cpuset *cur, struct cpuset *trial)
750 struct cpuset *c, *par;
761 /* On legacy hierarchy, we must be a subset of our parent cpuset. */
773 * validate_change() - Used to validate that any proposed cpuset change
776 * If we replaced the flag and mask values of the current cpuset
777 * (cur) with those values in the trial cpuset (trial), would
781 * 'cur' is the address of an actual, in-use cpuset. Operations
783 * cpuset in the list must use cur below, not trial.
792 static int validate_change(struct cpuset *cur, struct cpuset *trial)
795 struct cpuset *c, *par;
805 /* Remaining checks don't apply to root cpuset */
863 static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
869 update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
877 struct cpuset *root_cs)
879 struct cpuset *cp;
899 /* jump label reference count + the top-level cpuset */
925 * cp - cpuset pointer, used (together with pos_css) to perform a
933 * cpus_allowed of every cpuset marked is_sched_load_balance
944 * load balanced cpusets (using the array of cpuset pointers in
959 struct cpuset *cp; /* top-down scan of cpusets */
960 struct cpuset **csa; /* array of all cpuset ptrs */
961 int csn; /* how many cpuset ptrs in csa so far */
1041 struct cpuset *a = csa[i];
1045 struct cpuset *b = csa[j];
1050 struct cpuset *c = csa[k];
1077 struct cpuset *a = csa[i];
1102 struct cpuset *b = csa[j];
1133 static void dl_update_tasks_root_domain(struct cpuset *cs)
1151 struct cpuset *cs = NULL;
1198 * If the flag 'sched_load_balance' of any cpuset with non-empty
1199 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
1200 * which has that flag enabled, or if any cpuset with a non-empty
1211 struct cpuset *cs;
1273 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
1274 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
1278 * effective cpuset's. As this function is called with cpuset_mutex held,
1279 * cpuset membership stays stable. For top_cpuset, task_cpu_possible_mask()
1283 static void update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
1309 * compute_effective_cpumask - Compute the effective cpumask of the cpuset
1311 * @cs: the cpuset the need to recompute the new effective_cpus mask
1312 * @parent: the parent cpuset
1314 * The result is valid only if the given cpuset isn't a partition root.
1317 struct cpuset *cs, struct cpuset *parent)
1333 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1335 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
1343 static int update_partition_exclusive(struct cpuset *cs, int new_prs)
1364 static void update_partition_sd_lb(struct cpuset *cs, int old_prs)
1394 static bool tasks_nocpu_error(struct cpuset *parent, struct cpuset *cs,
1406 static void reset_partition_data(struct cpuset *cs)
1408 struct cpuset *parent = parent_cs(cs);
1447 * @parent: parent cpuset
1453 static bool partition_xcpus_add(int new_prs, struct cpuset *parent,
1479 * @parent: parent cpuset
1485 static bool partition_xcpus_del(int old_prs, struct cpuset *parent,
1534 * @cs: cpuset
1541 static bool compute_effective_exclusive_cpumask(struct cpuset *cs,
1544 struct cpuset *parent = parent_cs(cs);
1557 static inline bool is_remote_partition(struct cpuset *cs)
1562 static inline bool is_local_partition(struct cpuset *cs)
1568 * remote_partition_enable - Enable current cpuset as a remote partition root
1569 * @cs: the cpuset to update
1574 * Enable the current cpuset to become a remote partition root taking CPUs
1575 * directly from the top cpuset. cpuset_mutex must be held by the caller.
1577 static int remote_partition_enable(struct cpuset *cs, int new_prs,
1606 struct cpuset *parent = parent_cs(cs);
1623 * remote_partition_disable - Remove current cpuset from remote partition list
1624 * @cs: the cpuset to update
1631 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
1659 * @cs: the cpuset to be updated
1666 static void remote_cpus_update(struct cpuset *cs, struct cpumask *newmask,
1687 * left in the top cpuset.
1715 * @cs: the cpuset to be updated
1723 static void remote_partition_check(struct cpuset *cs, struct cpumask *newmask,
1726 struct cpuset *child, *next;
1740 * Since a cpuset must be removed from the remote children list
1742 * any change in cpuset status. RCU read lock isn't needed.
1775 * update_parent_effective_cpumask - update effective_cpus mask of parent cpuset
1776 * @cs: The cpuset that requests change in partition root state
1782 * For partcmd_enable*, the cpuset is being transformed from a non-partition
1784 * effective_xcpus not set) mask of the given cpuset will be taken away from
1788 * For partcmd_disable, the cpuset is being transformed from a partition
1794 * assumed to remain the same. The cpuset should either be a valid or invalid
1811 static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
1815 struct cpuset *parent = parent_cs(cs);
1973 struct cpuset *child;
2042 * Change the parent's effective_cpus & effective_xcpus (top cpuset
2097 * @cs: partition root cpuset
2111 static void compute_partition_effective_cpumask(struct cpuset *cs,
2115 struct cpuset *child;
2170 * @cs: the cpuset to consider
2174 * When configured cpumask is changed, the effective cpumasks of this cpuset
2181 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
2184 struct cpuset *cp;
2191 struct cpuset *parent = parent_cs(cp);
2197 * directly from top cpuset unless it is cs.
2272 * cpuset if the parent's effective_cpus changes.
2303 * The cpuset partition_root_state may become
2332 * from parent if current cpuset isn't a valid partition root
2346 * empty cpuset is changed, we need to rebuild sched domains.
2347 * On default hierarchy, the cpuset needs to be a partition
2367 * @parent: Parent cpuset
2368 * @cs: Current cpuset
2371 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
2374 struct cpuset *sibling;
2417 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
2418 * @cs: the cpuset to consider
2419 * @trialcs: trial cpuset
2420 * @buf: buffer of cpu numbers written to this cpuset
2422 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
2427 struct cpuset *parent = parent_cs(cs);
2437 * An empty cpus_allowed is ok only if the cpuset has no tasks.
2496 struct cpuset *cp;
2569 * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
2570 * @cs: the cpuset to consider
2571 * @trialcs: trial cpuset
2572 * @buf: buffer of cpu numbers written to this cpuset
2576 static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs,
2581 struct cpuset *parent = parent_cs(cs);
2757 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
2758 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
2761 * effective cpuset's. As this function is called with cpuset_mutex held,
2762 * cpuset membership stays stable.
2764 static void update_tasks_nodemask(struct cpuset *cs)
2811 /* We're done rebinding vmas to this cpuset's new mems_allowed. */
2817 * @cs: the cpuset to consider
2820 * When configured nodemask is changed, the effective nodemasks of this cpuset
2827 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
2829 struct cpuset *cp;
2834 struct cpuset *parent = parent_cs(cp);
2872 * of a cpuset. Needs to validate the request, update the
2873 * cpusets mems_allowed, and for each task in the cpuset,
2875 * mempolicies and if the cpuset is marked 'memory_migrate',
2879 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
2883 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
2898 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
2948 static int update_relax_domain_level(struct cpuset *cs, s64 val)
2966 * update_tasks_flags - update the spread flags of tasks in the cpuset.
2967 * @cs: the cpuset in which each task's spread flags needs to be changed
2970 * function is called with cpuset_mutex held, cpuset membership stays
2973 static void update_tasks_flags(struct cpuset *cs)
2987 * cs: the cpuset to update
2993 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
2996 struct cpuset *trialcs;
3036 * @cs: the cpuset to update
3042 static int update_prstate(struct cpuset *cs, int new_prs)
3045 struct cpuset *parent = parent_cs(cs);
3248 static struct cpuset *cpuset_attach_old_cs;
3251 * Check to see if a cpuset can accept a new task
3256 static int cpuset_can_attach_check(struct cpuset *cs)
3264 static void reset_migrate_dl_data(struct cpuset *cs)
3270 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
3274 struct cpuset *cs, *oldcs;
3286 /* Check to see if task is allowed in the cpuset */
3350 struct cpuset *cs;
3378 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
3402 struct cpuset *cs;
3403 struct cpuset *oldcs = cpuset_attach_old_cs;
3416 * In the default hierarchy, enabling cpuset in the child cgroups
3480 /* The various types of files and directories in a cpuset file system */
3507 struct cpuset *cs = css_cs(css);
3556 struct cpuset *cs = css_cs(css);
3585 struct cpuset *cs = css_cs(of_css(of));
3586 struct cpuset *trialcs;
3660 struct cpuset *cs = css_cs(seq_css(sf));
3701 struct cpuset *cs = css_cs(css);
3732 struct cpuset *cs = css_cs(css);
3747 struct cpuset *cs = css_cs(seq_css(seq));
3779 struct cpuset *cs = css_cs(of_css(of));
3959 .file_offset = offsetof(struct cpuset, partition_file),
3997 * cpuset_css_alloc - Allocate a cpuset css
3998 * @parent_css: Parent css of the control group that the new cpuset will be
4000 * Return: cpuset css on success, -ENOMEM on failure.
4002 * Allocate and initialize a new cpuset css, for non-NULL @parent_css, return
4003 * top cpuset css otherwise.
4008 struct cpuset *cs;
4038 struct cpuset *cs = css_cs(css);
4039 struct cpuset *parent = parent_cs(cs);
4040 struct cpuset *tmp_cs;
4117 * If the cpuset being removed has its flag 'sched_load_balance'
4123 * If the cpuset has the 'sched.partition' flag enabled, simulate
4129 struct cpuset *cs = css_cs(css);
4142 struct cpuset *parent = parent_cs(cs);
4157 struct cpuset *cs = css_cs(css);
4182 * In case the child is cloned into a cpuset different from its parent,
4187 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
4201 /* Check to see if task is allowed in the cpuset */
4226 struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
4245 * which could have been changed by cpuset just after it inherits the
4250 struct cpuset *cs;
4332 * or memory nodes, we need to walk over the cpuset hierarchy,
4334 * last CPU or node from a cpuset, then move the tasks in the empty
4335 * cpuset to its next-highest non-empty parent.
4337 static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
4339 struct cpuset *parent;
4342 * Find its next-highest non-empty parent, (top cpuset
4351 pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
4358 hotplug_update_tasks_legacy(struct cpuset *cs,
4372 * Don't call update_tasks_cpumask() if the cpuset becomes empty,
4386 * cpuset. Should be done outside any lock.
4396 hotplug_update_tasks(struct cpuset *cs,
4449 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
4450 * @cs: cpuset in interest
4457 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
4465 struct cpuset *parent;
4473 * is finished, so we won't attach a task to an empty cpuset.
4544 goto unlock; /* Hotplug doesn't affect this cpuset */
4561 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
4565 * changed and updates cpuset accordingly. The top_cpuset is always
4649 struct cpuset *cs;
4705 * Description: Finish top cpuset after cpu, node maps are initialized
4712 * cpuset_bind() call when v1 cpuset is mounted.
4726 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
4727 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
4730 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
4733 * tasks cpuset, except when the task is in the top cpuset.
4739 struct cpuset *cs;
4748 * Tasks in the top cpuset won't get update to their cpumasks
4822 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
4823 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
4825 * Description: Returns the nodemask_t mems_allowed of the cpuset
4828 * tasks cpuset.
4858 * mem_hardwall ancestor to the specified cpuset. Call holding
4860 * (an unusual configuration), then returns the root cpuset.
4862 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
4876 * node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
4881 * and do not allow allocations outside the current tasks cpuset
4884 * nearest enclosing hardwalled ancestor cpuset.
4891 * cpuset are short of memory, might require taking the callback_lock.
4895 * so no allocation on a node outside the cpuset is allowed (unless
4906 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
4911 struct cpuset *cs; /* current cpuset ancestors */
4948 * tasks in a cpuset with is_spread_page or is_spread_slab set),
5017 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
5019 * Description: Prints current's name, cpuset name, and cached copy of its
5029 pr_cont(",cpuset=");
5040 * cpuset file 'memory_pressure_enabled' in the root cpuset.
5046 * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
5049 * page reclaim efforts initiated by tasks in each cpuset.
5051 * This represents the rate at which some task in the cpuset
5057 * Display to user space in the per-cpuset read-only file
5060 * (direct) page reclaim by any task attached to the cpuset.
5073 * - Print tasks cpuset path into seq_file.
5074 * - Used for /proc/<pid>/cpuset.
5075 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
5076 * doesn't really matter if tsk->cpuset changes after we read it,