1/* $NetBSD: spl.S,v 1.49 2023/03/01 08:38:50 riastradh Exp $ */ 2 3/* 4 * Copyright (c) 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Frank van der Linden for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38/* 39 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc. 40 * All rights reserved. 41 * 42 * This code is derived from software contributed to The NetBSD Foundation 43 * by Charles M. Hannum and Andrew Doran. 44 * 45 * Redistribution and use in source and binary forms, with or without 46 * modification, are permitted provided that the following conditions 47 * are met: 48 * 1. Redistributions of source code must retain the above copyright 49 * notice, this list of conditions and the following disclaimer. 50 * 2. Redistributions in binary form must reproduce the above copyright 51 * notice, this list of conditions and the following disclaimer in the 52 * documentation and/or other materials provided with the distribution. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67#include "opt_ddb.h" 68#include "opt_kasan.h" 69#include "opt_kmsan.h" 70 71#define ALIGN_TEXT .align 16,0x90 72 73#include <machine/asm.h> 74#include <machine/trap.h> 75#include <machine/segments.h> 76#include <machine/frameasm.h> 77 78#include "assym.h" 79 80 .text 81 82/* 83 * int splraise(int s); 84 */ 85ENTRY(splraise) 86 movzbl CPUVAR(ILEVEL),%eax 87 cmpl %edi,%eax 88 cmoval %eax,%edi 89 movb %dil,CPUVAR(ILEVEL) 90 KMSAN_INIT_RET(4) 91 ret 92END(splraise) 93 94/* 95 * Xsoftintr() 96 * 97 * Switch to the LWP assigned to handle interrupts from the given 98 * source. We borrow the VM context from the interrupted LWP. 99 * 100 * On entry: 101 * 102 * %rax intrsource 103 * %r13 address to return to 104 */ 105IDTVEC(softintr) 106 /* set up struct switchframe */ 107 pushq $_C_LABEL(softintr_ret) 108 pushq %rbx 109 pushq %r12 110 pushq %r13 111 pushq %r14 112 pushq %r15 113 114 movb $IPL_HIGH,CPUVAR(ILEVEL) 115 movq CPUVAR(CURLWP),%r15 116 movq IS_LWP(%rax),%rdi /* switch to handler LWP */ 117 movq L_PCB(%rdi),%rdx 118 movq L_PCB(%r15),%rcx 119 /* 120 * Simple MOV to set curlwp to softlwp. See below on ordering 121 * required to restore softlwp like cpu_switchto. 122 * 123 * 1. Don't need store-before-store barrier because x86 is TSO. 124 * 125 * 2. Don't need store-before-load barrier because when we 126 * enter a softint lwp, it can't be holding any mutexes, so 127 * it can't release any until after it has acquired them, so 128 * we need not participate in the protocol with 129 * mutex_vector_enter barriers here. 130 * 131 * Hence no need for XCHG or barriers around MOV. 132 */ 133 movq %rdi,CPUVAR(CURLWP) 134 135#ifdef KASAN 136 /* clear the new stack */ 137 pushq %rax 138 pushq %rdx 139 pushq %rcx 140 callq _C_LABEL(kasan_softint) 141 popq %rcx 142 popq %rdx 143 popq %rax 144#endif 145 146#ifdef KMSAN 147 pushq %rax 148 pushq %rdx 149 pushq %rcx 150 callq _C_LABEL(kmsan_softint) 151 popq %rcx 152 popq %rdx 153 popq %rax 154#endif 155 156 /* save old context */ 157 movq %rsp,PCB_RSP(%rcx) 158 movq %rbp,PCB_RBP(%rcx) 159 160 /* switch to the new stack */ 161 movq PCB_RSP0(%rdx),%rsp 162 163 /* dispatch */ 164 STI(di) 165 movq %r15,%rdi /* interrupted LWP */ 166 movl IS_MAXLEVEL(%rax),%esi /* ipl to run at */ 167 call _C_LABEL(softint_dispatch)/* run handlers */ 168 CLI(di) 169 170 /* restore old context */ 171 movq L_PCB(%r15),%rcx 172 movq PCB_RSP(%rcx),%rsp 173 174 /* 175 * Use XCHG, not MOV, to coordinate mutex_exit on this CPU with 176 * mutex_vector_enter on another CPU. 177 * 178 * 1. Any prior mutex_exit by the softint must be visible to 179 * other CPUs before we restore curlwp on this one, 180 * requiring store-before-store ordering. 181 * 182 * (This is always guaranteed by the x86 memory model, TSO, 183 * but other architectures require a explicit barrier before 184 * the store to ci->ci_curlwp.) 185 * 186 * 2. Restoring curlwp must be visible on all other CPUs before 187 * any subsequent mutex_exit on this one can even test 188 * whether there might be waiters, requiring 189 * store-before-load ordering. 190 * 191 * (This is the only ordering x86 TSO ever requires any kind 192 * of barrier for -- in this case, we take advantage of the 193 * sequential consistency implied by XCHG to obviate the 194 * need for MFENCE or something.) 195 * 196 * See kern_mutex.c for details -- this is necessary for 197 * adaptive mutexes to detect whether the lwp is on the CPU in 198 * order to safely block without requiring atomic r/m/w in 199 * mutex_exit. See also cpu_switchto. 200 */ 201 xchgq %r15,CPUVAR(CURLWP) /* restore curlwp */ 202 popq %r15 /* unwind switchframe */ 203 addq $(5 * 8),%rsp 204 jmp *%r13 /* back to Xspllower/Xdoreti */ 205IDTVEC_END(softintr) 206 207/* 208 * softintr_ret() 209 * 210 * Trampoline function that gets returned to by cpu_switchto() when 211 * an interrupt handler blocks. On entry: 212 * 213 * %rax prevlwp from cpu_switchto() 214 */ 215ENTRY(softintr_ret) 216 incl CPUVAR(MTX_COUNT) /* re-adjust after mi_switch */ 217 CLI(ax) /* %rax not used by Xspllower/Xdoreti */ 218 jmp *%r13 /* back to Xspllower/Xdoreti */ 219END(softintr_ret) 220 221/* 222 * void softint_trigger(uintptr_t machdep); 223 * 224 * Software interrupt registration. 225 */ 226ENTRY(softint_trigger) 227 shlq $8,%rdi /* clear upper 8 bits */ 228 shrq $8,%rdi 229 orq %rdi,CPUVAR(IPENDING) /* atomic on local cpu */ 230 ret 231END(softint_trigger) 232 233/* 234 * Xrecurse_preempt() 235 * 236 * Handles preemption interrupts via Xspllower(). 237 */ 238IDTVEC(recurse_preempt) 239 movb $IPL_PREEMPT,CPUVAR(ILEVEL) 240 STI(di) 241 xorq %rdi,%rdi 242 KMSAN_INIT_ARG(8) 243 call _C_LABEL(kpreempt) 244 CLI(di) 245 jmp *%r13 /* back to Xspllower */ 246IDTVEC_END(recurse_preempt) 247 248/* 249 * Xresume_preempt() 250 * 251 * Handles preemption interrupts via Xdoreti(). 252 */ 253IDTVEC(resume_preempt) 254 movb $IPL_PREEMPT,CPUVAR(ILEVEL) 255 STI(ax) 256 testq $SEL_RPL,TF_CS(%rsp) 257 jnz 1f 258 movq TF_RIP(%rsp),%rdi 259 KMSAN_INIT_ARG(8) 260 call _C_LABEL(kpreempt) /* from kernel */ 261 CLI(ax) 262 jmp *%r13 /* back to Xdoreti */ 2631: 264 call _C_LABEL(preempt) /* from user */ 265 CLI(ax) 266 jmp *%r13 /* back to Xdoreti */ 267IDTVEC_END(resume_preempt) 268 269/* 270 * void spllower(int s); 271 */ 272ENTRY(spllower) 273 movzbl CPUVAR(ILEVEL),%edx 274 cmpl %edx,%edi /* new level is lower? */ 275 jae 1f 276 xorq %rcx,%rcx /* rcx: ci_ipending mask */ 277 notq %rcx 278 shrq $8,%rcx 279 movq %rdi,%r9 /* r9: shifted new level */ 280 shlq $56,%r9 2810: 282 movq CPUVAR(IPENDING),%rax 283 testq %rax,CPUVAR(IUNMASK)(,%rdi,8) /* deferred interrupts? */ 284 /* 285 * On the P4 this jump is cheaper than patching in junk 286 * using cmov. Is cmpxchg expensive if it fails? 287 */ 288 jnz 2f 289 movq %rax,%r8 290 andq %rcx,%r8 291 orq %r9,%r8 292 cmpxchgq %r8,CPUVAR(ISTATE) /* swap in new ilevel */ 293 jnz 0b 2941: 295 ret 2962: 297 jmp _C_LABEL(Xspllower) 298END(spllower) 299 300/* 301 * void Xspllower(int s); 302 * 303 * Process pending interrupts. 304 * 305 * Important registers: 306 * ebx - cpl 307 * r13 - address to resume loop at 308 * 309 * It is important that the bit scan instruction is bsr, it will get 310 * the highest 2 bits (currently the IPI and clock handlers) first, 311 * to avoid deadlocks where one CPU sends an IPI, another one is at 312 * splhigh() and defers it, lands in here via splx(), and handles 313 * a lower-prio one first, which needs to take the kernel lock --> 314 * the sending CPU will never see the that CPU accept the IPI 315 * (see pmap_tlb_shootnow). 316 */ 317IDTVEC(spllower) 318 pushq %rbx 319 pushq %r13 320 pushq %r12 321 movl %edi,%ebx 322 leaq 1f(%rip),%r13 /* address to resume loop at */ 3231: 324 movl %ebx,%eax /* get cpl */ 325 movq CPUVAR(IUNMASK)(,%rax,8),%rax 326 CLI(si) 327 andq CPUVAR(IPENDING),%rax /* any non-masked bits left? */ 328 jz 2f 329 bsrq %rax,%rax 330 btrq %rax,CPUVAR(IPENDING) 331 movq CPUVAR(ISOURCES)(,%rax,8),%rax 332 jmp *IS_RECURSE(%rax) 3332: 334 movb %bl,CPUVAR(ILEVEL) 335 STI(si) 336 popq %r12 337 popq %r13 338 popq %rbx 339 ret 340IDTVEC_END(spllower) 341 342/* 343 * void Xdoreti(void); 344 * 345 * Handle return from interrupt after device handler finishes. 346 * 347 * Important registers: 348 * ebx - cpl to restore 349 * r13 - address to resume loop at 350 */ 351IDTVEC(doreti) 352 popq %rbx /* get previous priority */ 353 decl CPUVAR(IDEPTH) 354 leaq 1f(%rip),%r13 3551: 356 movl %ebx,%eax 357 movq CPUVAR(IUNMASK)(,%rax,8),%rax 358 CLI(si) 359 andq CPUVAR(IPENDING),%rax 360 jz 2f 361 bsrq %rax,%rax /* slow, but not worth optimizing */ 362 btrq %rax,CPUVAR(IPENDING) 363 movq CPUVAR(ISOURCES)(,%rax,8),%rax 364 jmp *IS_RESUME(%rax) 3652: /* Check for ASTs on exit to user mode. */ 366 movb %bl,CPUVAR(ILEVEL) 3675: 368 testb $SEL_RPL,TF_CS(%rsp) 369 jz 6f 370.Ldoreti_checkast: 371 movq CPUVAR(CURLWP),%r14 372 CHECK_ASTPENDING(%r14) 373 je 3f 374 CLEAR_ASTPENDING(%r14) 375 STI(si) 376 movl $T_ASTFLT,TF_TRAPNO(%rsp) /* XXX undo later.. */ 377 /* Pushed T_ASTFLT into tf_trapno on entry. */ 378 movq %rsp,%rdi 379 KMSAN_INIT_ARG(8) 380 call _C_LABEL(trap) 381 CLI(si) 382 jmp .Ldoreti_checkast 3833: 384 CHECK_DEFERRED_SWITCH 385 jnz 9f 386 HANDLE_DEFERRED_FPU 3876: 388 INTRFASTEXIT 3899: 390 STI(si) 391 call _C_LABEL(do_pmap_load) 392 CLI(si) 393 jmp .Ldoreti_checkast /* recheck ASTs */ 394IDTVEC_END(doreti) 395