1/* $NetBSD: mptramp.S,v 1.28 2019/11/14 16:23:52 maxv Exp $ */ 2 3/* 4 * Copyright (c) 2000, 2016 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by RedBack Networks Inc. (Author: Bill Sommerfeld), and Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64/* 65 * MP startup ... 66 * the stuff from cpu_spinup_trampoline to mp_startup is copied into the 67 * first 640 KB. 68 * 69 * We startup the processors now when the kthreads become ready. 70 * The steps are: 71 * 1) Get the processors running kernel-code from a special 72 * page-table and stack page, do chip identification. 73 * 2) halt the processors waiting for them to be enabled 74 * by a idle-thread 75 */ 76 77#include "assym.h" 78#include "opt_kcsan.h" 79#include "opt_kmsan.h" 80#include <machine/asm.h> 81#include <machine/specialreg.h> 82#include <machine/segments.h> 83#include <machine/mpbiosvar.h> 84#include <machine/i82489reg.h> 85#include <machine/gdt.h> 86 87#define _TRMP_LABEL(a) a = . - _C_LABEL(cpu_spinup_trampoline) + MP_TRAMPOLINE 88 89/* 90 * A smp_data structure is packed at the end of the trampoline page. The stack 91 * is right below this structure. 92 */ 93#define SMP_DATA (MP_TRAMPOLINE + PAGE_SIZE - 3 * 4) 94#define SMP_DATA_STACK (SMP_DATA + 0 * 4) 95#define SMP_DATA_LARGE (SMP_DATA + 0 * 4) 96#define SMP_DATA_NOX (SMP_DATA + 1 * 4) 97#define SMP_DATA_PDIR (SMP_DATA + 2 * 4) 98 99 .global _C_LABEL(cpu_spinup_trampoline) 100 .global _C_LABEL(cpu_spinup_trampoline_end) 101 .global _C_LABEL(cpu_hatch) 102 103 .text 104 .align 4,0x0 105 .code16 106/* XXX ENTRY() */ 107LABEL(cpu_spinup_trampoline) 108 cli 109 xorw %ax,%ax 110 movw %ax,%ds 111 movw %ax,%es 112 movw %ax,%ss 113 114 /* load flat descriptor table */ 115#ifdef __clang__ 116 lgdt (mptramp_gdt32_desc) 117#else 118 data32 addr32 lgdt (mptramp_gdt32_desc) 119#endif 120 121 /* enable protected mode */ 122 movl %cr0,%eax 123 orl $CR0_PE,%eax 124 movl %eax,%cr0 125 ljmpl $0x8,$mp_startup 126 127_TRMP_LABEL(mp_startup) 128 .code32 129 130 movl $0x10,%eax /* data segment */ 131 movw %ax,%ds 132 movw %ax,%ss 133 movw %ax,%es 134 movw %ax,%fs 135 movw %ax,%gs 136 137 /* bootstrap stack end */ 138 movl $SMP_DATA_STACK,%esp 139 140 /* First, reset the PSL. */ 141 pushl $PSL_MBO 142 popfl 143 144 /* Enable PAE, SSE, and PSE if available */ 145 movl %cr4,%eax 146 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 147 movl $SMP_DATA_LARGE,%ecx 148 movl (%ecx),%ecx 149 orl %ecx,%ecx 150 jz .Lno_PSE 151 orl $CR4_PSE,%eax 152.Lno_PSE: 153 movl %eax,%cr4 154 155 /* 156 * Set Long Mode Enable in EFER. Also enable the syscall extensions, 157 * and NOX if available. 158 */ 159 movl $MSR_EFER,%ecx 160 rdmsr 161 xorl %eax,%eax 162 orl $(EFER_LME|EFER_SCE),%eax 163 movl $SMP_DATA_NOX,%ebx 164 movl (%ebx),%ebx 165 cmpl $0,%ebx 166 je .Lno_NOX 167 orl $(EFER_NXE),%eax 168.Lno_NOX: 169 wrmsr 170 171 /* Load %cr3. */ 172 movl $SMP_DATA_PDIR,%ecx 173 movl (%ecx),%ecx /* guaranteed < 4G */ 174 movl %ecx,%cr3 /* load PTD addr into MMU */ 175 176 /* Enable paging and the rest of it. */ 177 movl %cr0,%eax 178 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 179 movl %eax,%cr0 180 181 jmp .Lmptramp_compat 182.Lmptramp_compat: 183 184 movl $GSEL(GDATA_SEL, SEL_KPL),%eax /* switch to new segment */ 185 movl %eax,%ds 186 movl %eax,%es 187 movl %eax,%ss 188 189 movl $mptramp_gdt64_desc,%eax 190 lgdt (%eax) 191 movl $mptramp_jmp64,%eax 192 ljmp *(%eax) 193 194_TRMP_LABEL(mptramp_jmp64) 195 .long mptramp_longmode 196 .word GSEL(GCODE_SEL, SEL_KPL) 197 198#define GDT_LIMIT 0x17 /* 23 = 3 * 8 - 1 */ 199_TRMP_LABEL(mptramp_gdt32) 200 .quad 0x0000000000000000 201 .quad 0x00cf9f000000ffff /* CS */ 202 .quad 0x00cf93000000ffff /* DS */ 203_TRMP_LABEL(mptramp_gdt32_desc) 204 .word GDT_LIMIT 205 .long mptramp_gdt32 206 207_TRMP_LABEL(mptramp_gdt64) 208 .quad 0x0000000000000000 209 .quad 0x00af9a000000ffff /* CS */ 210 .quad 0x00cf92000000ffff /* DS */ 211_TRMP_LABEL(mptramp_gdt64_desc) 212 .word GDT_LIMIT 213 .long mptramp_gdt64 214#undef GDT_LIMIT 215 216_TRMP_LABEL(mptramp_longmode) 217 .code64 218 movabsq $_C_LABEL(cpu_spinup_trampoline_end),%rax 219 jmp *%rax 220 221 222_C_LABEL(cpu_spinup_trampoline_end): /* end of code copied to MP_TRAMPOLINE */ 223 224 /* Wait until BP has done init sequence. */ 2251: 226 movq _C_LABEL(cpu_starting),%rdi 227 pause 228 testq %rdi,%rdi 229 jz 1b 230 231 movq CPU_INFO_IDLELWP(%rdi),%rsi 232 movq L_PCB(%rsi),%rsi 233 234 movq PCB_RSP(%rsi),%rsp 235 movq PCB_RBP(%rsi),%rbp 236 237 movq CPU_INFO_GDT(%rdi),%rax 238 movw $(MAXGDTSIZ-1),-10(%rsp) 239 movq %rax,-8(%rsp) 240 lgdt -10(%rsp) 241 242 /* Switch address space. */ 243 movq PCB_CR3(%rsi),%rax 244 movq %rax,%cr3 245 246 movl PCB_CR0(%rsi),%eax 247 movq %rax,%cr0 248 249#if defined(KCSAN) || defined(KMSAN) 250 /* 251 * The C instrumentation uses GS.base, so initialize it right now. It 252 * gets re-initialized later, that's fine. 253 */ 254 movl $MSR_GSBASE,%ecx 255 movq %rdi,%rax 256 movq %rdi,%rdx 257 shrq $32,%rdx 258 wrmsr 259#endif 260 261 call _C_LABEL(cpu_hatch) 262END(cpu_spinup_trampoline) 263 264