1/*	$NetBSD: intr.c,v 1.56 2020/11/01 20:58:38 christos Exp $	*/
2
3/*
4 * Copyright (c) 2008-2010, 2015 Antti Kantee.  All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.56 2020/11/01 20:58:38 christos Exp $");
30
31#include <sys/param.h>
32#include <sys/atomic.h>
33#include <sys/cpu.h>
34#include <sys/kernel.h>
35#include <sys/kmem.h>
36#include <sys/kthread.h>
37#include <sys/malloc.h>
38#include <sys/intr.h>
39#include <sys/timetc.h>
40
41#include <rump-sys/kern.h>
42
43#include <rump/rumpuser.h>
44
45/*
46 * Interrupt simulator.  It executes hardclock() and softintrs.
47 */
48
49#define SI_MPSAFE 0x01
50#define SI_KILLME 0x02
51
52struct softint_percpu;
53struct softint {
54	void (*si_func)(void *);
55	void *si_arg;
56	int si_flags;
57	int si_level;
58
59	struct softint_percpu *si_entry; /* [0,ncpu-1] */
60};
61
62struct softint_percpu {
63	struct softint *sip_parent;
64	bool sip_onlist;
65	bool sip_onlist_cpu;
66
67	TAILQ_ENTRY(softint_percpu) sip_entries;	/* scheduled */
68	TAILQ_ENTRY(softint_percpu) sip_entries_cpu;	/* to be scheduled */
69};
70
71struct softint_lev {
72	struct rumpuser_cv *si_cv;
73	TAILQ_HEAD(, softint_percpu) si_pending;
74};
75
76static TAILQ_HEAD(, softint_percpu) sicpupending \
77    = TAILQ_HEAD_INITIALIZER(sicpupending);
78static struct rumpuser_mtx *sicpumtx;
79static struct rumpuser_cv *sicpucv;
80
81kcondvar_t lbolt; /* Oh Kath Ra */
82
83static int ncpu_final;
84
85void noclock(void); void noclock(void) {return;}
86__strong_alias(sched_schedclock,noclock);
87__strong_alias(cpu_initclocks,noclock);
88__strong_alias(addupc_intr,noclock);
89__strong_alias(sched_tick,noclock);
90__strong_alias(setstatclockrate,noclock);
91
92/*
93 * clock "interrupt"
94 */
95static void
96doclock(void *noarg)
97{
98	struct timespec thetick, curclock;
99	struct clockframe *clkframe;
100	int64_t sec;
101	long nsec;
102	int error;
103	struct cpu_info *ci = curcpu();
104
105	error = rumpuser_clock_gettime(RUMPUSER_CLOCK_ABSMONO, &sec, &nsec);
106	if (error)
107		panic("clock: cannot get monotonic time");
108
109	curclock.tv_sec = sec;
110	curclock.tv_nsec = nsec;
111	thetick.tv_sec = 0;
112	thetick.tv_nsec = 1000000000/hz;
113
114	/* generate dummy clockframe for hardclock to consume */
115	clkframe = rump_cpu_makeclockframe();
116
117	for (;;) {
118		int lbolt_ticks = 0;
119
120		hardclock(clkframe);
121		if (CPU_IS_PRIMARY(ci)) {
122			if (++lbolt_ticks >= hz) {
123				lbolt_ticks = 0;
124				cv_broadcast(&lbolt);
125			}
126		}
127
128		error = rumpuser_clock_sleep(RUMPUSER_CLOCK_ABSMONO,
129		    curclock.tv_sec, curclock.tv_nsec);
130		if (error) {
131			panic("rumpuser_clock_sleep failed with error %d",
132			    error);
133		}
134		timespecadd(&curclock, &thetick, &curclock);
135	}
136}
137
138/*
139 * Soft interrupt execution thread.  This thread is pinned to the
140 * same CPU that scheduled the interrupt, so we don't need to do
141 * lock against si_lvl.
142 */
143static void
144sithread(void *arg)
145{
146	struct softint_percpu *sip;
147	struct softint *si;
148	void (*func)(void *) = NULL;
149	void *funarg;
150	bool mpsafe;
151	int mylevel = (uintptr_t)arg;
152	struct softint_lev *si_lvlp, *si_lvl;
153	struct cpu_data *cd = &curcpu()->ci_data;
154
155	si_lvlp = cd->cpu_softcpu;
156	si_lvl = &si_lvlp[mylevel];
157
158	for (;;) {
159		if (!TAILQ_EMPTY(&si_lvl->si_pending)) {
160			sip = TAILQ_FIRST(&si_lvl->si_pending);
161			si = sip->sip_parent;
162
163			func = si->si_func;
164			funarg = si->si_arg;
165			mpsafe = si->si_flags & SI_MPSAFE;
166
167			sip->sip_onlist = false;
168			TAILQ_REMOVE(&si_lvl->si_pending, sip, sip_entries);
169			if (si->si_flags & SI_KILLME) {
170				softint_disestablish(si);
171				continue;
172			}
173		} else {
174			rump_schedlock_cv_wait(si_lvl->si_cv);
175			continue;
176		}
177
178		if (!mpsafe)
179			KERNEL_LOCK(1, curlwp);
180		func(funarg);
181		if (!mpsafe)
182			KERNEL_UNLOCK_ONE(curlwp);
183	}
184
185	panic("sithread unreachable");
186}
187
188/*
189 * Helper for softint_schedule_cpu()
190 */
191static void
192sithread_cpu_bouncer(void *arg)
193{
194	struct lwp *me;
195
196	me = curlwp;
197	me->l_pflag |= LP_BOUND;
198
199	rump_unschedule();
200	for (;;) {
201		struct softint_percpu *sip;
202		struct softint *si;
203		struct cpu_info *ci;
204		unsigned int cidx;
205
206		rumpuser_mutex_enter_nowrap(sicpumtx);
207		while (TAILQ_EMPTY(&sicpupending)) {
208			rumpuser_cv_wait_nowrap(sicpucv, sicpumtx);
209		}
210		sip = TAILQ_FIRST(&sicpupending);
211		TAILQ_REMOVE(&sicpupending, sip, sip_entries_cpu);
212		sip->sip_onlist_cpu = false;
213		rumpuser_mutex_exit(sicpumtx);
214
215		/*
216		 * ok, now figure out which cpu we need the softint to
217		 * be handled on
218		 */
219		si = sip->sip_parent;
220		cidx = sip - si->si_entry;
221		ci = cpu_lookup(cidx);
222		me->l_target_cpu = ci;
223
224		/* schedule ourselves there, and then schedule the softint */
225		rump_schedule();
226		KASSERT(curcpu() == ci);
227		softint_schedule(si);
228		rump_unschedule();
229	}
230	panic("sithread_cpu_bouncer unreasonable");
231}
232
233static kmutex_t sithr_emtx;
234static unsigned int sithr_est;
235static int sithr_canest;
236
237/*
238 * Create softint handler threads when the softint for each respective
239 * level is established for the first time.  Most rump kernels don't
240 * need at least half of the softint levels, so on-demand saves bootstrap
241 * time and memory resources.  Note, though, that this routine may be
242 * called before it's possible to call kthread_create().  Creation of
243 * those softints (SOFTINT_CLOCK, as of writing this) will be deferred
244 * to until softint_init() is called for the main CPU.
245 */
246static void
247sithread_establish(int level)
248{
249	int docreate, rv;
250	int lvlbit = 1<<level;
251	int i;
252
253	KASSERT((level & ~SOFTINT_LVLMASK) == 0);
254	if (__predict_true(sithr_est & lvlbit))
255		return;
256
257	mutex_enter(&sithr_emtx);
258	docreate = (sithr_est & lvlbit) == 0 && sithr_canest;
259	sithr_est |= lvlbit;
260	mutex_exit(&sithr_emtx);
261
262	if (docreate) {
263		for (i = 0; i < ncpu_final; i++) {
264			if ((rv = kthread_create(PRI_NONE,
265			    KTHREAD_MPSAFE | KTHREAD_INTR,
266			    cpu_lookup(i), sithread, (void *)(uintptr_t)level,
267			    NULL, "rsi%d/%d", i, level)) != 0)
268				panic("softint thread create failed: %d", rv);
269		}
270	}
271}
272
273void
274rump_intr_init(int numcpu)
275{
276
277	cv_init(&lbolt, "oh kath ra");
278	mutex_init(&sithr_emtx, MUTEX_DEFAULT, IPL_NONE);
279	ncpu_final = numcpu;
280}
281
282void
283softint_init(struct cpu_info *ci)
284{
285	struct cpu_data *cd = &ci->ci_data;
286	struct softint_lev *slev;
287	int rv, i;
288
289	if (!rump_threads)
290		return;
291
292	slev = kmem_alloc(sizeof(struct softint_lev) * SOFTINT_COUNT, KM_SLEEP);
293	for (i = 0; i < SOFTINT_COUNT; i++) {
294		rumpuser_cv_init(&slev[i].si_cv);
295		TAILQ_INIT(&slev[i].si_pending);
296	}
297	cd->cpu_softcpu = slev;
298
299	/* overloaded global init ... */
300	/* XXX: should be done the last time we are called */
301	if (ci->ci_index == 0) {
302		int sithr_swap;
303
304		/* pretend that we have our own for these */
305		stathz = 1;
306		schedhz = 1;
307		profhz = 1;
308
309		initclocks();
310
311		/* create deferred softint threads */
312		mutex_enter(&sithr_emtx);
313		sithr_swap = sithr_est;
314		sithr_est = 0;
315		sithr_canest = 1;
316		mutex_exit(&sithr_emtx);
317		for (i = 0; i < SOFTINT_COUNT; i++) {
318			if (sithr_swap & (1<<i))
319				sithread_establish(i);
320		}
321	}
322
323	/* well, not really a "soft" interrupt ... */
324	if ((rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
325	    ci, doclock, NULL, NULL, "rumpclk%d", ci->ci_index)) != 0)
326		panic("clock thread creation failed: %d", rv);
327
328	/* not one either, but at least a softint helper */
329	rumpuser_mutex_init(&sicpumtx, RUMPUSER_MTX_SPIN);
330	rumpuser_cv_init(&sicpucv);
331	if ((rv = kthread_create(PRI_NONE, KTHREAD_MPSAFE,
332	    NULL, sithread_cpu_bouncer, NULL, NULL, "sipbnc")) != 0)
333		panic("softint cpu bouncer creation failed: %d", rv);
334}
335
336void *
337softint_establish(u_int flags, void (*func)(void *), void *arg)
338{
339	struct softint *si;
340	struct softint_percpu *sip;
341	int level = flags & SOFTINT_LVLMASK;
342	int i;
343
344	si = malloc(sizeof(*si), M_TEMP, M_WAITOK);
345	si->si_func = func;
346	si->si_arg = arg;
347	si->si_flags = flags & SOFTINT_MPSAFE ? SI_MPSAFE : 0;
348	si->si_level = level;
349	KASSERT(si->si_level < SOFTINT_COUNT);
350	si->si_entry = malloc(sizeof(*si->si_entry) * ncpu_final,
351	    M_TEMP, M_WAITOK | M_ZERO);
352	for (i = 0; i < ncpu_final; i++) {
353		sip = &si->si_entry[i];
354		sip->sip_parent = si;
355	}
356	sithread_establish(level);
357
358	return si;
359}
360
361static struct softint_percpu *
362sitosip(struct softint *si, struct cpu_info *ci)
363{
364
365	return &si->si_entry[ci->ci_index];
366}
367
368/*
369 * Soft interrupts bring two choices.  If we are running with thread
370 * support enabled, defer execution, otherwise execute in place.
371 */
372
373void
374softint_schedule(void *arg)
375{
376	struct softint *si = arg;
377	struct cpu_info *ci = curcpu();
378	struct softint_percpu *sip = sitosip(si, ci);
379	struct cpu_data *cd = &ci->ci_data;
380	struct softint_lev *si_lvl = cd->cpu_softcpu;
381
382	if (!rump_threads) {
383		si->si_func(si->si_arg);
384	} else {
385		if (!sip->sip_onlist) {
386			TAILQ_INSERT_TAIL(&si_lvl[si->si_level].si_pending,
387			    sip, sip_entries);
388			sip->sip_onlist = true;
389		}
390	}
391}
392
393/*
394 * Like softint_schedule(), except schedule softint to be handled on
395 * the core designated by ci_tgt instead of the core the call is made on.
396 *
397 * Unlike softint_schedule(), the performance is not important
398 * (unless ci_tgt == curcpu): high-performance rump kernel I/O stacks
399 * should arrange data to already be on the right core at the driver
400 * layer.
401 */
402void
403softint_schedule_cpu(void *arg, struct cpu_info *ci_tgt)
404{
405	struct softint *si = arg;
406	struct cpu_info *ci_cur = curcpu();
407	struct softint_percpu *sip;
408
409	KASSERT(rump_threads);
410
411	/* preferred case (which can be optimized some day) */
412	if (ci_cur == ci_tgt) {
413		softint_schedule(si);
414		return;
415	}
416
417	/*
418	 * no?  then it's softint turtles all the way down
419	 */
420
421	sip = sitosip(si, ci_tgt);
422	rumpuser_mutex_enter_nowrap(sicpumtx);
423	if (sip->sip_onlist_cpu) {
424		rumpuser_mutex_exit(sicpumtx);
425		return;
426	}
427	TAILQ_INSERT_TAIL(&sicpupending, sip, sip_entries_cpu);
428	sip->sip_onlist_cpu = true;
429	rumpuser_cv_signal(sicpucv);
430	rumpuser_mutex_exit(sicpumtx);
431}
432
433/*
434 * flimsy disestablish: should wait for softints to finish.
435 */
436void
437softint_disestablish(void *cook)
438{
439	struct softint *si = cook;
440	int i;
441
442	for (i = 0; i < ncpu_final; i++) {
443		struct softint_percpu *sip;
444
445		sip = &si->si_entry[i];
446		if (sip->sip_onlist) {
447			si->si_flags |= SI_KILLME;
448			return;
449		}
450	}
451	free(si->si_entry, M_TEMP);
452	free(si, M_TEMP);
453}
454
455void
456rump_softint_run(struct cpu_info *ci)
457{
458	struct cpu_data *cd = &ci->ci_data;
459	struct softint_lev *si_lvl = cd->cpu_softcpu;
460	int i;
461
462	if (!rump_threads)
463		return;
464
465	for (i = 0; i < SOFTINT_COUNT; i++) {
466		if (!TAILQ_EMPTY(&si_lvl[i].si_pending))
467			rump_schedlock_cv_signal(ci, si_lvl[i].si_cv);
468	}
469}
470
471bool
472cpu_intr_p(void)
473{
474
475	return false;
476}
477