1/*	$NetBSD: subr_interrupt.c,v 1.5 2021/12/10 20:36:04 andvar Exp $	*/
2
3/*
4 * Copyright (c) 2015 Internet Initiative Japan Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.5 2021/12/10 20:36:04 andvar Exp $");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/errno.h>
36#include <sys/cpu.h>
37#include <sys/interrupt.h>
38#include <sys/intr.h>
39#include <sys/kcpuset.h>
40#include <sys/kmem.h>
41#include <sys/proc.h>
42#include <sys/xcall.h>
43#include <sys/sysctl.h>
44
45#include <sys/conf.h>
46#include <sys/intrio.h>
47#include <sys/kauth.h>
48
49#include <machine/limits.h>
50
51#ifdef INTR_DEBUG
52#define DPRINTF(msg) printf msg
53#else
54#define DPRINTF(msg)
55#endif
56
57static struct intrio_set kintrio_set = { "\0", NULL, 0 };
58
59#define UNSET_NOINTR_SHIELD	0
60#define SET_NOINTR_SHIELD	1
61
62static void
63interrupt_shield_xcall(void *arg1, void *arg2)
64{
65	struct cpu_info *ci;
66	struct schedstate_percpu *spc;
67	int s, shield;
68
69	ci = arg1;
70	shield = (int)(intptr_t)arg2;
71	spc = &ci->ci_schedstate;
72
73	s = splsched();
74	if (shield == UNSET_NOINTR_SHIELD)
75		spc->spc_flags &= ~SPCF_NOINTR;
76	else if (shield == SET_NOINTR_SHIELD)
77		spc->spc_flags |= SPCF_NOINTR;
78	splx(s);
79}
80
81/*
82 * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags.
83 */
84static int
85interrupt_shield(u_int cpu_idx, int shield)
86{
87	struct cpu_info *ci;
88	struct schedstate_percpu *spc;
89
90	KASSERT(mutex_owned(&cpu_lock));
91
92	ci = cpu_lookup(cpu_idx);
93	if (ci == NULL)
94		return EINVAL;
95
96	spc = &ci->ci_schedstate;
97	if (shield == UNSET_NOINTR_SHIELD) {
98		if ((spc->spc_flags & SPCF_NOINTR) == 0)
99			return 0;
100	} else if (shield == SET_NOINTR_SHIELD) {
101		if ((spc->spc_flags & SPCF_NOINTR) != 0)
102			return 0;
103	}
104
105	if (ci == curcpu() || !mp_online) {
106		interrupt_shield_xcall(ci, (void *)(intptr_t)shield);
107	} else {
108		uint64_t where;
109		where = xc_unicast(0, interrupt_shield_xcall, ci,
110			(void *)(intptr_t)shield, ci);
111		xc_wait(where);
112	}
113
114	spc->spc_lastmod = time_second;
115	return 0;
116}
117
118/*
119 * Move all assigned interrupts from "cpu_idx" to the other cpu as possible.
120 * The destination cpu is the lowest cpuid of available cpus.
121 * If there are no available cpus, give up to move interrupts.
122 */
123static int
124interrupt_avert_intr(u_int cpu_idx)
125{
126	kcpuset_t *cpuset;
127	struct intrids_handler *ii_handler;
128	intrid_t *ids;
129	int error = 0, i, nids;
130
131	kcpuset_create(&cpuset, true);
132	kcpuset_set(cpuset, cpu_idx);
133
134	ii_handler = interrupt_construct_intrids(cpuset);
135	if (ii_handler == NULL) {
136		error = EINVAL;
137		goto out;
138	}
139	nids = ii_handler->iih_nids;
140	if (nids == 0) {
141		error = 0;
142		goto destruct_out;
143	}
144
145	interrupt_get_available(cpuset);
146	kcpuset_clear(cpuset, cpu_idx);
147	if (kcpuset_iszero(cpuset)) {
148		DPRINTF(("%s: no available cpu\n", __func__));
149		error = ENOENT;
150		goto destruct_out;
151	}
152
153	ids = ii_handler->iih_intrids;
154	for (i = 0; i < nids; i++) {
155		error = interrupt_distribute_handler(ids[i], cpuset, NULL);
156		if (error)
157			break;
158	}
159
160 destruct_out:
161	interrupt_destruct_intrids(ii_handler);
162 out:
163	kcpuset_destroy(cpuset);
164	return error;
165}
166
167/*
168 * Return actual intrio_list_line size.
169 * intrio_list_line size is variable by ncpu.
170 */
171static size_t
172interrupt_intrio_list_line_size(void)
173{
174
175	return sizeof(struct intrio_list_line) +
176		sizeof(struct intrio_list_line_cpu) * (ncpu - 1);
177}
178
179/*
180 * Return the size of interrupts list data on success.
181 * Reterun 0 on failed.
182 */
183static int
184interrupt_intrio_list_size(size_t *ilsize)
185{
186	struct intrids_handler *ii_handler;
187
188	*ilsize = 0;
189
190	/* buffer header */
191	*ilsize += sizeof(struct intrio_list);
192
193	/* il_line body */
194	ii_handler = interrupt_construct_intrids(kcpuset_running);
195	if (ii_handler == NULL)
196		return EOPNOTSUPP;
197	*ilsize += interrupt_intrio_list_line_size() * ii_handler->iih_nids;
198
199	interrupt_destruct_intrids(ii_handler);
200	return 0;
201}
202
203/*
204 * Set intrctl list data to "il", and return list structure bytes.
205 * If error occurred, return <0.
206 * If "data" == NULL, simply return list structure bytes.
207 */
208static int
209interrupt_intrio_list(struct intrio_list *il, size_t ilsize)
210{
211	struct intrio_list_line *illine;
212	kcpuset_t *assigned, *avail;
213	struct intrids_handler *ii_handler;
214	intrid_t *ids;
215	u_int cpu_idx;
216	int nids, intr_idx, error, line_size;
217
218	illine = (struct intrio_list_line *)
219	    ((char *)il + sizeof(struct intrio_list));
220	il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il);
221
222	kcpuset_create(&avail, true);
223	interrupt_get_available(avail);
224	kcpuset_create(&assigned, true);
225
226	ii_handler = interrupt_construct_intrids(kcpuset_running);
227	if (ii_handler == NULL) {
228		DPRINTF(("%s: interrupt_construct_intrids() failed\n",
229		    __func__));
230		error = EOPNOTSUPP;
231		goto out;
232	}
233
234	line_size = interrupt_intrio_list_line_size();
235	/* ensure interrupts are not added after interrupt_intrio_list_size() */
236	nids = ii_handler->iih_nids;
237	ids = ii_handler->iih_intrids;
238	if (ilsize < sizeof(struct intrio_list) + line_size * nids) {
239		DPRINTF(("%s: interrupts are added during execution.\n",
240		    __func__));
241		error = EAGAIN;
242		goto destruct_out;
243	}
244
245	for (intr_idx = 0; intr_idx < nids; intr_idx++) {
246		char devname[INTRDEVNAMEBUF];
247
248		strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF);
249		interrupt_get_devname(ids[intr_idx], devname, sizeof(devname));
250		strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF);
251
252		interrupt_get_assigned(ids[intr_idx], assigned);
253		for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) {
254			struct intrio_list_line_cpu *illcpu =
255			    &illine->ill_cpu[cpu_idx];
256
257			illcpu->illc_assigned =
258			    kcpuset_isset(assigned, cpu_idx);
259			illcpu->illc_count =
260			    interrupt_get_count(ids[intr_idx], cpu_idx);
261		}
262
263		illine = (struct intrio_list_line *)
264		    ((char *)illine + line_size);
265	}
266
267	error = 0;
268	il->il_version = INTRIO_LIST_VERSION;
269	il->il_ncpus = ncpu;
270	il->il_nintrs = nids;
271	il->il_linesize = line_size;
272	il->il_bufsize = ilsize;
273
274 destruct_out:
275	interrupt_destruct_intrids(ii_handler);
276 out:
277	kcpuset_destroy(assigned);
278	kcpuset_destroy(avail);
279
280	return error;
281}
282
283/*
284 * "intrctl list" entry
285 */
286static int
287interrupt_intrio_list_sysctl(SYSCTLFN_ARGS)
288{
289	int error;
290	void *buf;
291	size_t ilsize;
292
293	if (oldlenp == NULL)
294		return EINVAL;
295
296	if ((error = interrupt_intrio_list_size(&ilsize)) != 0)
297		return error;
298
299	/*
300	 * If oldp == NULL, the sysctl(8) caller process want to get the size of
301	 * intrctl list data only.
302	 */
303	if (oldp == NULL) {
304		*oldlenp = ilsize;
305		return 0;
306	}
307
308	/*
309	 * If oldp != NULL, the sysctl(8) caller process want to get both the
310	 * size and the contents of intrctl list data.
311	 */
312	if (*oldlenp < ilsize)
313		return ENOMEM;
314
315	buf = kmem_zalloc(ilsize, KM_SLEEP);
316	if ((error = interrupt_intrio_list(buf, ilsize)) != 0)
317		goto out;
318
319	error = copyout(buf, oldp, ilsize);
320 out:
321	kmem_free(buf, ilsize);
322	return error;
323}
324
325/*
326 * "intrctl affinity" entry
327 */
328static int
329interrupt_set_affinity_sysctl(SYSCTLFN_ARGS)
330{
331	struct sysctlnode node;
332	struct intrio_set *iset;
333	cpuset_t *ucpuset;
334	kcpuset_t *kcpuset;
335	int error;
336
337	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR,
338	    KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL);
339	if (error)
340		return EPERM;
341
342	node = *rnode;
343	iset = (struct intrio_set *)node.sysctl_data;
344
345	error = sysctl_lookup(SYSCTLFN_CALL(&node));
346	if (error != 0 || newp == NULL)
347		return error;
348
349	ucpuset = iset->cpuset;
350	kcpuset_create(&kcpuset, true);
351	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
352	if (error)
353		goto out;
354	if (kcpuset_iszero(kcpuset)) {
355		error = EINVAL;
356		goto out;
357	}
358
359	error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL);
360
361 out:
362	kcpuset_destroy(kcpuset);
363	return error;
364}
365
366/*
367 * "intrctl intr" entry
368 */
369static int
370interrupt_intr_sysctl(SYSCTLFN_ARGS)
371{
372	struct sysctlnode node;
373	struct intrio_set *iset;
374	cpuset_t *ucpuset;
375	kcpuset_t *kcpuset;
376	int error;
377	u_int cpu_idx;
378
379	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
380	    KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
381	if (error)
382		return EPERM;
383
384	node = *rnode;
385	iset = (struct intrio_set *)node.sysctl_data;
386
387	error = sysctl_lookup(SYSCTLFN_CALL(&node));
388	if (error != 0 || newp == NULL)
389		return error;
390
391	ucpuset = iset->cpuset;
392	kcpuset_create(&kcpuset, true);
393	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
394	if (error)
395		goto out;
396	if (kcpuset_iszero(kcpuset)) {
397		error = EINVAL;
398		goto out;
399	}
400
401	cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
402
403	mutex_enter(&cpu_lock);
404	error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD);
405	mutex_exit(&cpu_lock);
406
407 out:
408	kcpuset_destroy(kcpuset);
409	return error;
410}
411
412/*
413 * "intrctl nointr" entry
414 */
415static int
416interrupt_nointr_sysctl(SYSCTLFN_ARGS)
417{
418	struct sysctlnode node;
419	struct intrio_set *iset;
420	cpuset_t *ucpuset;
421	kcpuset_t *kcpuset;
422	int error;
423	u_int cpu_idx;
424
425	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
426	    KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
427	if (error)
428		return EPERM;
429
430	node = *rnode;
431	iset = (struct intrio_set *)node.sysctl_data;
432
433	error = sysctl_lookup(SYSCTLFN_CALL(&node));
434	if (error != 0 || newp == NULL)
435		return error;
436
437	ucpuset = iset->cpuset;
438	kcpuset_create(&kcpuset, true);
439	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
440	if (error)
441		goto out;
442	if (kcpuset_iszero(kcpuset)) {
443		error = EINVAL;
444		goto out;
445	}
446
447	cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
448
449	mutex_enter(&cpu_lock);
450	error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD);
451	mutex_exit(&cpu_lock);
452	if (error)
453		goto out;
454
455	error = interrupt_avert_intr(cpu_idx);
456
457 out:
458	kcpuset_destroy(kcpuset);
459	return error;
460}
461
462SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup")
463{
464	const struct sysctlnode *node = NULL;
465
466	sysctl_createv(clog, 0, NULL, &node,
467		       CTLFLAG_PERMANENT, CTLTYPE_NODE,
468		       "intr", SYSCTL_DESCR("Interrupt options"),
469		       NULL, 0, NULL, 0,
470		       CTL_KERN, CTL_CREATE, CTL_EOL);
471
472	sysctl_createv(clog, 0, &node, NULL,
473		       CTLFLAG_PERMANENT, CTLTYPE_STRUCT,
474		       "list", SYSCTL_DESCR("intrctl list"),
475		       interrupt_intrio_list_sysctl, 0, NULL,
476		        0, CTL_CREATE, CTL_EOL);
477
478	sysctl_createv(clog, 0, &node, NULL,
479		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
480		       "affinity", SYSCTL_DESCR("set affinity"),
481		       interrupt_set_affinity_sysctl, 0, &kintrio_set,
482		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
483
484	sysctl_createv(clog, 0, &node, NULL,
485		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
486		       "intr", SYSCTL_DESCR("set intr"),
487		       interrupt_intr_sysctl, 0, &kintrio_set,
488		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
489
490	sysctl_createv(clog, 0, &node, NULL,
491		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
492		       "nointr", SYSCTL_DESCR("set nointr"),
493		       interrupt_nointr_sysctl, 0, &kintrio_set,
494		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
495}
496