kernel/sched/core.c

11554  * @cid_lock: Guarantee forward-progress of cid allocation.
11563  * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
11565  * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
11569  * of a cid allocation.
11574  * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
11580  * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
11581  * used by a task. That would lead to concurrent allocation of the cid and
11597  * per-mm/cpu cid value.
11618  * when a thread is actively using the cid (property (1)).
11646  * it will move its state to UNSET, which clears the percpu cid perhaps
11658  * because this would UNSET a cid which is actively used.
11680 	 * If the migrated task has no last cid, or if the current
11681 	 * task on src rq uses the cid, it means the source cid does not need
11686 	src_cid = READ_ONCE(src_pcpu_cid->cid);
11719 	 * Attempt to clear the source cpu cid to move it to the destination
11723 	if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
11727 	 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
11730 	 * per-mm/cpu cid.
11732 	 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
11736 	 * load of per-mm/cpu cid.
11749 			 * no point in moving this cid to the destination cpu.
11759 	if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
11766  * Interrupts are disabled, which keeps the window of cid ownership without the
11786 	 * Move the src cid if the dst cid is unset. This keeps id
11790 	 * If destination cid is already set, we may have to just clear
11791 	 * the src cid to ensure compactness in frequent migrations
11794 	 * It is not useful to clear the src cid when the number of threads is
11800 	dst_cid = READ_ONCE(dst_pcpu_cid->cid);
11819 	WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
11827 	int cid, lazy_cid;
11829 	cid = READ_ONCE(pcpu_cid->cid);
11830 	if (!mm_cid_is_valid(cid))
11834 	 * Clear the cpu cid if it is set to keep cid allocation compact.  If
11836 	 * mm, the next task using this mm will reallocate its cid on context
11839 	lazy_cid = mm_cid_set_lazy_put(cid);
11840 	if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
11844 	 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
11847 	 * per-mm/cpu cid.
11849 	 * The implicit barrier after cmpxchg per-mm/cpu cid before loading
11853 	 * load of per-mm/cpu cid.
11868 	 * The cid is unused, so it can be unset.
11869 	 * Disable interrupts to keep the window of cid ownership without rq
11873 		if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
11874 			__mm_cid_put(mm, cid);
11894 	 * snapshot associated with this cid if an active task using the mm is
11914 	int cid;
11917 	cid = READ_ONCE(pcpu_cid->cid);
11918 	if (!mm_cid_is_valid(cid) || cid < weight)
12008 	 * Store t->mm_cid_active before loading per-mm/cpu cid.
12030 	 * Store t->mm_cid_active before loading per-mm/cpu cid.
12052 		 * Store t->mm_cid_active before loading per-mm/cpu cid.