1/*	$NetBSD: kern_kthread.c,v 1.49 2023/09/23 14:40:42 ad Exp $	*/
2
3/*-
4 * Copyright (c) 1998, 1999, 2007, 2009, 2019, 2023
5 *     The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10 * NASA Ames Research Center, and by Andrew Doran.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 *    notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 *    notice, this list of conditions and the following disclaimer in the
19 *    documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/cdefs.h>
35__KERNEL_RCSID(0, "$NetBSD: kern_kthread.c,v 1.49 2023/09/23 14:40:42 ad Exp $");
36
37#include <sys/param.h>
38#include <sys/cpu.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/kthread.h>
42#include <sys/mutex.h>
43#include <sys/sched.h>
44#include <sys/kmem.h>
45#include <sys/msan.h>
46
47#include <uvm/uvm_extern.h>
48
49static kmutex_t		kthread_lock;
50static kcondvar_t	kthread_cv;
51
52void
53kthread_sysinit(void)
54{
55
56	mutex_init(&kthread_lock, MUTEX_DEFAULT, IPL_NONE);
57	cv_init(&kthread_cv, "kthrwait");
58}
59
60/*
61 * kthread_create: create a kernel thread, that is, system-only LWP.
62 */
63int
64kthread_create(pri_t pri, int flag, struct cpu_info *ci,
65    void (*func)(void *), void *arg, lwp_t **lp, const char *fmt, ...)
66{
67	lwp_t *l;
68	vaddr_t uaddr;
69	int error, lc;
70	va_list ap;
71
72	KASSERT((flag & KTHREAD_INTR) == 0 || (flag & KTHREAD_MPSAFE) != 0);
73
74	uaddr = uvm_uarea_system_alloc(
75	   (flag & (KTHREAD_INTR|KTHREAD_IDLE)) == KTHREAD_IDLE ? ci : NULL);
76	if (uaddr == 0) {
77		return ENOMEM;
78	}
79	kmsan_orig((void *)uaddr, USPACE, KMSAN_TYPE_POOL, __RET_ADDR);
80	if ((flag & KTHREAD_TS) != 0) {
81		lc = SCHED_OTHER;
82	} else {
83		lc = SCHED_RR;
84	}
85
86	error = lwp_create(&lwp0, &proc0, uaddr, LWP_DETACHED, NULL,
87	    0, func, arg, &l, lc, &lwp0.l_sigmask, &lwp0.l_sigstk);
88	if (error) {
89		uvm_uarea_system_free(uaddr);
90		return error;
91	}
92	if (fmt != NULL) {
93		l->l_name = kmem_alloc(MAXCOMLEN, KM_SLEEP);
94		va_start(ap, fmt);
95		vsnprintf(l->l_name, MAXCOMLEN, fmt, ap);
96		va_end(ap);
97	}
98
99	/*
100	 * Set parameters.
101	 */
102	if (pri == PRI_NONE) {
103		if ((flag & KTHREAD_TS) != 0) {
104			/* Maximum user priority level. */
105			pri = MAXPRI_USER;
106		} else {
107			/* Minimum kernel priority level. */
108			pri = PRI_KTHREAD;
109		}
110	}
111	mutex_enter(proc0.p_lock);
112	lwp_lock(l);
113	lwp_changepri(l, pri);
114	if (ci != NULL) {
115		if (ci != l->l_cpu) {
116			lwp_unlock_to(l, ci->ci_schedstate.spc_lwplock);
117			lwp_lock(l);
118			l->l_cpu = ci;
119		}
120		l->l_pflag |= LP_BOUND;
121	}
122
123	if ((flag & KTHREAD_MUSTJOIN) != 0) {
124		KASSERT(lp != NULL);
125		l->l_pflag |= LP_MUSTJOIN;
126	}
127	if ((flag & KTHREAD_INTR) != 0) {
128		l->l_pflag |= LP_INTR;
129	}
130	if ((flag & KTHREAD_MPSAFE) == 0) {
131		l->l_pflag &= ~LP_MPSAFE;
132	}
133
134	/*
135	 * Set the new LWP running, unless the caller has requested
136	 * otherwise.
137	 */
138	KASSERT(l->l_stat == LSIDL);
139	if ((flag & KTHREAD_IDLE) == 0) {
140		setrunnable(l);
141		/* LWP now unlocked */
142	} else {
143		lwp_unlock(l);
144	}
145	mutex_exit(proc0.p_lock);
146
147	/* All done! */
148	if (lp != NULL) {
149		*lp = l;
150	}
151	return 0;
152}
153
154/*
155 * Cause a kernel thread to exit.  Assumes the exiting thread is the
156 * current context.
157 */
158void
159kthread_exit(int ecode)
160{
161	const char *name;
162	lwp_t *l = curlwp;
163
164	/* If the kernel lock is held, we need to drop it now. */
165	if ((l->l_pflag & LP_MPSAFE) == 0) {
166		KERNEL_UNLOCK_LAST(l);
167	}
168
169	/* We can't do much with the exit code, so just report it. */
170	if (ecode != 0) {
171		if ((name = l->l_name) == NULL)
172			name = "unnamed";
173		printf("WARNING: kthread `%s' (%d) exits with status %d\n",
174		    name, l->l_lid, ecode);
175	}
176
177	/* Barrier for joining. */
178	if (l->l_pflag & LP_MUSTJOIN) {
179		bool *exitedp;
180
181		mutex_enter(&kthread_lock);
182		while ((exitedp = l->l_private) == NULL) {
183			cv_wait(&kthread_cv, &kthread_lock);
184		}
185		KASSERT(!*exitedp);
186		*exitedp = true;
187		cv_broadcast(&kthread_cv);
188		mutex_exit(&kthread_lock);
189	}
190
191	/* And exit.. */
192	lwp_exit(l);
193	panic("kthread_exit");
194}
195
196/*
197 * Wait for a kthread to exit, as pthread_join().
198 */
199int
200kthread_join(lwp_t *l)
201{
202	bool exited = false;
203
204	KASSERT((l->l_flag & LW_SYSTEM) != 0);
205	KASSERT((l->l_pflag & LP_MUSTJOIN) != 0);
206
207	/*
208	 * - Ask the kthread to write to `exited'.
209	 * - After this, touching l is forbidden -- it may be freed.
210	 * - Wait until the kthread has written to `exited'.
211	 */
212	mutex_enter(&kthread_lock);
213	KASSERT(l->l_private == NULL);
214	l->l_private = &exited;
215	cv_broadcast(&kthread_cv);
216	while (!exited) {
217		cv_wait(&kthread_cv, &kthread_lock);
218	}
219	mutex_exit(&kthread_lock);
220
221	return 0;
222}
223
224/*
225 * kthread_fpu_enter()
226 *
227 *	Allow the current lwp, which must be a kthread, to use the FPU.
228 *	Return a cookie that must be passed to kthread_fpu_exit when
229 *	done.  Must be used only in thread context.  Recursive -- you
230 *	can call kthread_fpu_enter several times in a row as long as
231 *	you pass the cookies in reverse order to kthread_fpu_exit.
232 */
233int
234kthread_fpu_enter(void)
235{
236	struct lwp *l = curlwp;
237	int s;
238
239	KASSERTMSG(!cpu_intr_p(),
240	    "%s is not allowed in interrupt context", __func__);
241	KASSERTMSG(!cpu_softintr_p(),
242	    "%s is not allowed in interrupt context", __func__);
243
244	/*
245	 * Remember whether this thread already had FPU access, and
246	 * mark this thread as having FPU access.
247	 */
248	lwp_lock(l);
249	KASSERTMSG(l->l_flag & LW_SYSTEM,
250	    "%s is allowed only in kthreads", __func__);
251	s = l->l_flag & LW_SYSTEM_FPU;
252	l->l_flag |= LW_SYSTEM_FPU;
253	lwp_unlock(l);
254
255	/* Take MD steps to enable the FPU if necessary.  */
256	if (s == 0)
257		kthread_fpu_enter_md();
258
259	return s;
260}
261
262/*
263 * kthread_fpu_exit(s)
264 *
265 *	Restore the current lwp's FPU access to what it was before the
266 *	matching call to kthread_fpu_enter() that returned s.  Must be
267 *	used only in thread context.
268 */
269void
270kthread_fpu_exit(int s)
271{
272	struct lwp *l = curlwp;
273
274	KASSERT(s == (s & LW_SYSTEM_FPU));
275	KASSERTMSG(!cpu_intr_p(),
276	    "%s is not allowed in interrupt context", __func__);
277	KASSERTMSG(!cpu_softintr_p(),
278	    "%s is not allowed in interrupt context", __func__);
279
280	lwp_lock(l);
281	KASSERTMSG(l->l_flag & LW_SYSTEM,
282	    "%s is allowed only in kthreads", __func__);
283	KASSERT(l->l_flag & LW_SYSTEM_FPU);
284	l->l_flag ^= s ^ LW_SYSTEM_FPU;
285	lwp_unlock(l);
286
287	/* Take MD steps to zero and disable the FPU if necessary.  */
288	if (s == 0)
289		kthread_fpu_exit_md();
290}
291