1/*	$NetBSD: spl.S,v 1.49 2023/03/01 08:38:50 riastradh Exp $	*/
2
3/*
4 * Copyright (c) 2003 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Frank van der Linden for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 *    must display the following acknowledgement:
19 *      This product includes software developed for the NetBSD Project by
20 *      Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 *    or promote products derived from this software without specific prior
23 *    written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38/*
39 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
40 * All rights reserved.
41 *
42 * This code is derived from software contributed to The NetBSD Foundation
43 * by Charles M. Hannum and Andrew Doran.
44 *
45 * Redistribution and use in source and binary forms, with or without
46 * modification, are permitted provided that the following conditions
47 * are met:
48 * 1. Redistributions of source code must retain the above copyright
49 *    notice, this list of conditions and the following disclaimer.
50 * 2. Redistributions in binary form must reproduce the above copyright
51 *    notice, this list of conditions and the following disclaimer in the
52 *    documentation and/or other materials provided with the distribution.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
57 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
64 * POSSIBILITY OF SUCH DAMAGE.
65 */
66
67#include "opt_ddb.h"
68#include "opt_kasan.h"
69#include "opt_kmsan.h"
70
71#define ALIGN_TEXT	.align 16,0x90
72
73#include <machine/asm.h>
74#include <machine/trap.h>
75#include <machine/segments.h>
76#include <machine/frameasm.h>
77
78#include "assym.h"
79
80	.text
81
82/*
83 * int splraise(int s);
84 */
85ENTRY(splraise)
86	movzbl	CPUVAR(ILEVEL),%eax
87	cmpl	%edi,%eax
88	cmoval	%eax,%edi
89	movb	%dil,CPUVAR(ILEVEL)
90	KMSAN_INIT_RET(4)
91	ret
92END(splraise)
93
94/*
95 * Xsoftintr()
96 *
97 * Switch to the LWP assigned to handle interrupts from the given
98 * source.  We borrow the VM context from the interrupted LWP.
99 *
100 * On entry:
101 *
102 *	%rax		intrsource
103 *	%r13		address to return to
104 */
105IDTVEC(softintr)
106	/* set up struct switchframe */
107	pushq	$_C_LABEL(softintr_ret)
108	pushq	%rbx
109	pushq	%r12
110	pushq	%r13
111	pushq	%r14
112	pushq	%r15
113
114	movb	$IPL_HIGH,CPUVAR(ILEVEL)
115	movq	CPUVAR(CURLWP),%r15
116	movq	IS_LWP(%rax),%rdi	/* switch to handler LWP */
117	movq	L_PCB(%rdi),%rdx
118	movq	L_PCB(%r15),%rcx
119	/*
120	 * Simple MOV to set curlwp to softlwp.  See below on ordering
121	 * required to restore softlwp like cpu_switchto.
122	 *
123	 * 1. Don't need store-before-store barrier because x86 is TSO.
124	 *
125	 * 2. Don't need store-before-load barrier because when we
126	 *    enter a softint lwp, it can't be holding any mutexes, so
127	 *    it can't release any until after it has acquired them, so
128	 *    we need not participate in the protocol with
129	 *    mutex_vector_enter barriers here.
130	 *
131	 * Hence no need for XCHG or barriers around MOV.
132	 */
133	movq	%rdi,CPUVAR(CURLWP)
134
135#ifdef KASAN
136	/* clear the new stack */
137	pushq	%rax
138	pushq	%rdx
139	pushq	%rcx
140	callq	_C_LABEL(kasan_softint)
141	popq	%rcx
142	popq	%rdx
143	popq	%rax
144#endif
145
146#ifdef KMSAN
147	pushq	%rax
148	pushq	%rdx
149	pushq	%rcx
150	callq	_C_LABEL(kmsan_softint)
151	popq	%rcx
152	popq	%rdx
153	popq	%rax
154#endif
155
156	/* save old context */
157	movq	%rsp,PCB_RSP(%rcx)
158	movq	%rbp,PCB_RBP(%rcx)
159
160	/* switch to the new stack */
161	movq	PCB_RSP0(%rdx),%rsp
162
163	/* dispatch */
164	STI(di)
165	movq	%r15,%rdi		/* interrupted LWP */
166	movl	IS_MAXLEVEL(%rax),%esi	/* ipl to run at */
167	call	_C_LABEL(softint_dispatch)/* run handlers */
168	CLI(di)
169
170	/* restore old context */
171	movq	L_PCB(%r15),%rcx
172	movq	PCB_RSP(%rcx),%rsp
173
174	/*
175	 * Use XCHG, not MOV, to coordinate mutex_exit on this CPU with
176	 * mutex_vector_enter on another CPU.
177	 *
178	 * 1. Any prior mutex_exit by the softint must be visible to
179	 *    other CPUs before we restore curlwp on this one,
180	 *    requiring store-before-store ordering.
181	 *
182	 *    (This is always guaranteed by the x86 memory model, TSO,
183	 *    but other architectures require a explicit barrier before
184	 *    the store to ci->ci_curlwp.)
185	 *
186	 * 2. Restoring curlwp must be visible on all other CPUs before
187	 *    any subsequent mutex_exit on this one can even test
188	 *    whether there might be waiters, requiring
189	 *    store-before-load ordering.
190	 *
191	 *    (This is the only ordering x86 TSO ever requires any kind
192	 *    of barrier for -- in this case, we take advantage of the
193	 *    sequential consistency implied by XCHG to obviate the
194	 *    need for MFENCE or something.)
195	 *
196	 * See kern_mutex.c for details -- this is necessary for
197	 * adaptive mutexes to detect whether the lwp is on the CPU in
198	 * order to safely block without requiring atomic r/m/w in
199	 * mutex_exit.  See also cpu_switchto.
200	 */
201	xchgq	%r15,CPUVAR(CURLWP)	/* restore curlwp */
202	popq	%r15			/* unwind switchframe */
203	addq	$(5 * 8),%rsp
204	jmp	*%r13			/* back to Xspllower/Xdoreti */
205IDTVEC_END(softintr)
206
207/*
208 * softintr_ret()
209 *
210 * Trampoline function that gets returned to by cpu_switchto() when
211 * an interrupt handler blocks.  On entry:
212 *
213 *	%rax		prevlwp from cpu_switchto()
214 */
215ENTRY(softintr_ret)
216	incl	CPUVAR(MTX_COUNT)	/* re-adjust after mi_switch */
217	CLI(ax)		/* %rax not used by Xspllower/Xdoreti */
218	jmp	*%r13			/* back to Xspllower/Xdoreti */
219END(softintr_ret)
220
221/*
222 * void softint_trigger(uintptr_t machdep);
223 *
224 * Software interrupt registration.
225 */
226ENTRY(softint_trigger)
227	shlq	$8,%rdi			/* clear upper 8 bits */
228	shrq	$8,%rdi
229	orq	%rdi,CPUVAR(IPENDING)	/* atomic on local cpu */
230	ret
231END(softint_trigger)
232
233/*
234 * Xrecurse_preempt()
235 *
236 * Handles preemption interrupts via Xspllower().
237 */
238IDTVEC(recurse_preempt)
239	movb	$IPL_PREEMPT,CPUVAR(ILEVEL)
240	STI(di)
241	xorq	%rdi,%rdi
242	KMSAN_INIT_ARG(8)
243	call	_C_LABEL(kpreempt)
244	CLI(di)
245	jmp	*%r13			/* back to Xspllower */
246IDTVEC_END(recurse_preempt)
247
248/*
249 * Xresume_preempt()
250 *
251 * Handles preemption interrupts via Xdoreti().
252 */
253IDTVEC(resume_preempt)
254	movb	$IPL_PREEMPT,CPUVAR(ILEVEL)
255	STI(ax)
256	testq	$SEL_RPL,TF_CS(%rsp)
257	jnz	1f
258	movq	TF_RIP(%rsp),%rdi
259	KMSAN_INIT_ARG(8)
260	call	_C_LABEL(kpreempt)	/* from kernel */
261	CLI(ax)
262	jmp	*%r13			/* back to Xdoreti */
2631:
264	call	_C_LABEL(preempt)	/* from user */
265	CLI(ax)
266	jmp	*%r13			/* back to Xdoreti */
267IDTVEC_END(resume_preempt)
268
269/*
270 * void spllower(int s);
271 */
272ENTRY(spllower)
273	movzbl	CPUVAR(ILEVEL),%edx
274	cmpl	%edx,%edi			/* new level is lower? */
275	jae	1f
276	xorq	%rcx,%rcx			/* rcx: ci_ipending mask */
277	notq	%rcx
278	shrq	$8,%rcx
279	movq	%rdi,%r9			/* r9: shifted new level */
280	shlq	$56,%r9
2810:
282	movq	CPUVAR(IPENDING),%rax
283	testq	%rax,CPUVAR(IUNMASK)(,%rdi,8)	/* deferred interrupts? */
284	/*
285	 * On the P4 this jump is cheaper than patching in junk
286	 * using cmov.  Is cmpxchg expensive if it fails?
287	 */
288	jnz	2f
289	movq	%rax,%r8
290	andq	%rcx,%r8
291	orq	%r9,%r8
292	cmpxchgq %r8,CPUVAR(ISTATE)		/* swap in new ilevel */
293	jnz	0b
2941:
295	ret
2962:
297	jmp	_C_LABEL(Xspllower)
298END(spllower)
299
300/*
301 * void Xspllower(int s);
302 *
303 * Process pending interrupts.
304 *
305 * Important registers:
306 *   ebx - cpl
307 *   r13 - address to resume loop at
308 *
309 * It is important that the bit scan instruction is bsr, it will get
310 * the highest 2 bits (currently the IPI and clock handlers) first,
311 * to avoid deadlocks where one CPU sends an IPI, another one is at
312 * splhigh() and defers it, lands in here via splx(), and handles
313 * a lower-prio one first, which needs to take the kernel lock -->
314 * the sending CPU will never see the that CPU accept the IPI
315 * (see pmap_tlb_shootnow).
316 */
317IDTVEC(spllower)
318	pushq	%rbx
319	pushq	%r13
320	pushq	%r12
321	movl	%edi,%ebx
322	leaq	1f(%rip),%r13		/* address to resume loop at */
3231:
324	movl	%ebx,%eax		/* get cpl */
325	movq	CPUVAR(IUNMASK)(,%rax,8),%rax
326	CLI(si)
327	andq	CPUVAR(IPENDING),%rax	/* any non-masked bits left? */
328	jz	2f
329	bsrq	%rax,%rax
330	btrq	%rax,CPUVAR(IPENDING)
331	movq	CPUVAR(ISOURCES)(,%rax,8),%rax
332	jmp	*IS_RECURSE(%rax)
3332:
334	movb	%bl,CPUVAR(ILEVEL)
335	STI(si)
336	popq	%r12
337	popq	%r13
338	popq	%rbx
339	ret
340IDTVEC_END(spllower)
341
342/*
343 * void Xdoreti(void);
344 *
345 * Handle return from interrupt after device handler finishes.
346 *
347 * Important registers:
348 *   ebx - cpl to restore
349 *   r13 - address to resume loop at
350 */
351IDTVEC(doreti)
352	popq	%rbx			/* get previous priority */
353	decl	CPUVAR(IDEPTH)
354	leaq	1f(%rip),%r13
3551:
356	movl    %ebx,%eax
357	movq	CPUVAR(IUNMASK)(,%rax,8),%rax
358	CLI(si)
359	andq	CPUVAR(IPENDING),%rax
360	jz	2f
361	bsrq	%rax,%rax		/* slow, but not worth optimizing */
362	btrq	%rax,CPUVAR(IPENDING)
363	movq	CPUVAR(ISOURCES)(,%rax,8),%rax
364	jmp	*IS_RESUME(%rax)
3652:	/* Check for ASTs on exit to user mode. */
366	movb	%bl,CPUVAR(ILEVEL)
3675:
368	testb	$SEL_RPL,TF_CS(%rsp)
369	jz	6f
370.Ldoreti_checkast:
371	movq	CPUVAR(CURLWP),%r14
372	CHECK_ASTPENDING(%r14)
373	je	3f
374	CLEAR_ASTPENDING(%r14)
375	STI(si)
376	movl	$T_ASTFLT,TF_TRAPNO(%rsp)	/* XXX undo later.. */
377	/* Pushed T_ASTFLT into tf_trapno on entry. */
378	movq	%rsp,%rdi
379	KMSAN_INIT_ARG(8)
380	call	_C_LABEL(trap)
381	CLI(si)
382	jmp	.Ldoreti_checkast
3833:
384	CHECK_DEFERRED_SWITCH
385	jnz	9f
386	HANDLE_DEFERRED_FPU
3876:
388	INTRFASTEXIT
3899:
390	STI(si)
391	call	_C_LABEL(do_pmap_load)
392	CLI(si)
393	jmp	.Ldoreti_checkast		/* recheck ASTs */
394IDTVEC_END(doreti)
395