1/* 2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 57/* 58 */ 59 60#include <kern/cpu_number.h> 61#include <kern/kalloc.h> 62#include <kern/cpu_data.h> 63#include <mach/mach_types.h> 64#include <mach/machine.h> 65#include <mach/vm_map.h> 66#include <vm/vm_kern.h> 67#include <vm/vm_map.h> 68 69#include <i386/mp_desc.h> 70#include <i386/lock.h> 71#include <i386/misc_protos.h> 72#include <i386/mp.h> 73#include <i386/pmap.h> 74#include <i386/machine_check.h> 75 76#include <kern/misc_protos.h> 77 78#include <mach_kdb.h> 79 80/* 81 * The i386 needs an interrupt stack to keep the PCB stack from being 82 * overrun by interrupts. All interrupt stacks MUST lie at lower addresses 83 * than any thread`s kernel stack. 84 */ 85 86/* 87 * First cpu`s interrupt stack. 88 */ 89extern uint32_t low_intstack[]; /* bottom */ 90extern uint32_t low_eintstack[]; /* top */ 91 92/* 93 * Per-cpu data area pointers. 94 * The master cpu (cpu 0) has its data area statically allocated; 95 * others are allocated dynamically and this array is updated at runtime. 96 */ 97cpu_data_t cpu_data_master = { 98 .cpu_this = &cpu_data_master, 99 .cpu_nanotime = &rtc_nanotime_info, 100 .cpu_is64bit = FALSE, 101 .cpu_int_stack_top = (vm_offset_t) low_eintstack, 102 }; 103cpu_data_t *cpu_data_ptr[MAX_CPUS] = { [0] &cpu_data_master }; 104 105decl_simple_lock_data(,cpu_lock); /* protects real_ncpus */ 106unsigned int real_ncpus = 1; 107unsigned int max_ncpus = MAX_CPUS; 108 109extern void *hi_remap_text; 110#define HI_TEXT(lo_text) \ 111 (((uint32_t)&lo_text - (uint32_t)&hi_remap_text) + HIGH_MEM_BASE) 112 113extern void hi_sysenter(void); 114extern void hi64_sysenter(void); 115extern void hi64_syscall(void); 116 117/* 118 * Multiprocessor i386/i486 systems use a separate copy of the 119 * GDT, IDT, LDT, and kernel TSS per processor. The first three 120 * are separate to avoid lock contention: the i386 uses locked 121 * memory cycles to access the descriptor tables. The TSS is 122 * separate since each processor needs its own kernel stack, 123 * and since using a TSS marks it busy. 124 */ 125 126/* 127 * Allocate and initialize the per-processor descriptor tables. 128 */ 129 130struct fake_descriptor ldt_desc_pattern = { 131 (unsigned int) 0, 132 LDTSZ_MIN * sizeof(struct fake_descriptor) - 1, 133 0, 134 ACC_P|ACC_PL_K|ACC_LDT 135}; 136 137struct fake_descriptor tss_desc_pattern = { 138 (unsigned int) 0, 139 sizeof(struct i386_tss) - 1, 140 0, 141 ACC_P|ACC_PL_K|ACC_TSS 142}; 143 144struct fake_descriptor cpudata_desc_pattern = { 145 (unsigned int) 0, 146 sizeof(cpu_data_t)-1, 147 SZ_32, 148 ACC_P|ACC_PL_K|ACC_DATA_W 149}; 150 151struct fake_descriptor userwindow_desc_pattern = { 152 (unsigned int) 0, 153 ((NBPDE * NCOPY_WINDOWS) / PAGE_SIZE) - 1, 154 SZ_32 | SZ_G, 155 ACC_P|ACC_PL_U|ACC_DATA_W 156}; 157 158struct fake_descriptor physwindow_desc_pattern = { 159 (unsigned int) 0, 160 PAGE_SIZE - 1, 161 SZ_32, 162 ACC_P|ACC_PL_K|ACC_DATA_W 163}; 164 165/* 166 * This is the expanded, 64-bit variant of the kernel LDT descriptor. 167 * When switching to 64-bit mode this replaces KERNEL_LDT entry 168 * and the following empty slot. This enables the LDT to be referenced 169 * in the uber-space remapping window on the kernel. 170 */ 171struct fake_descriptor64 kernel_ldt_desc64 = { 172 FAKE_UBER64(&master_ldt), 173 LDTSZ_MIN*sizeof(struct fake_descriptor)-1, 174 0, 175 ACC_P|ACC_PL_K|ACC_LDT, 176 0 177}; 178 179/* 180 * This is the expanded, 64-bit variant of the kernel TSS descriptor. 181 * It is follows pattern of the KERNEL_LDT. 182 */ 183struct fake_descriptor64 kernel_tss_desc64 = { 184 FAKE_UBER64(&master_ktss64), 185 sizeof(struct x86_64_tss)-1, 186 0, 187 ACC_P|ACC_PL_K|ACC_TSS, 188 0 189}; 190 191void 192cpu_desc_init( 193 cpu_data_t *cdp, 194 boolean_t is_boot_cpu) 195{ 196 cpu_desc_table_t *cdt = cdp->cpu_desc_tablep; 197 cpu_desc_index_t *cdi = &cdp->cpu_desc_index; 198 199 if (is_boot_cpu) { 200 /* 201 * Master CPU uses the tables built at boot time. 202 * Just set the index pointers to the high shared-mapping space. 203 * Note that the sysenter stack uses empty space above the ktss 204 * in the HIGH_FIXED_KTSS page. In this case we don't map the 205 * the real master_sstk in low memory. 206 */ 207 cdi->cdi_ktss = (struct i386_tss *) 208 pmap_index_to_virt(HIGH_FIXED_KTSS) ; 209 cdi->cdi_sstk = (vm_offset_t) (cdi->cdi_ktss + 1) + 210 (vm_offset_t) &master_sstk.top - 211 (vm_offset_t) &master_sstk; 212#if MACH_KDB 213 cdi->cdi_dbtss = (struct i386_tss *) 214 pmap_index_to_virt(HIGH_FIXED_DBTSS); 215#endif /* MACH_KDB */ 216 cdi->cdi_gdt = (struct fake_descriptor *) 217 pmap_index_to_virt(HIGH_FIXED_GDT); 218 cdi->cdi_idt = (struct fake_descriptor *) 219 pmap_index_to_virt(HIGH_FIXED_IDT); 220 cdi->cdi_ldt = (struct fake_descriptor *) 221 pmap_index_to_virt(HIGH_FIXED_LDT_BEGIN); 222 } else { 223 224 vm_offset_t cpu_hi_desc; 225 226 cpu_hi_desc = pmap_cpu_high_shared_remap(cdp->cpu_number, 227 HIGH_CPU_DESC, 228 (vm_offset_t) cdt, 1); 229 230 /* 231 * Per-cpu GDT, IDT, LDT, KTSS descriptors are allocated in one 232 * block (cpu_desc_table) and double-mapped into high shared space 233 * in one page window. 234 * Also, a transient stack for the fast sysenter path. The top of 235 * which is set at context switch time to point to the PCB using 236 * the high address. 237 */ 238 cdi->cdi_gdt = (struct fake_descriptor *) (cpu_hi_desc + 239 offsetof(cpu_desc_table_t, gdt[0])); 240 cdi->cdi_idt = (struct fake_descriptor *) (cpu_hi_desc + 241 offsetof(cpu_desc_table_t, idt[0])); 242 cdi->cdi_ktss = (struct i386_tss *) (cpu_hi_desc + 243 offsetof(cpu_desc_table_t, ktss)); 244 cdi->cdi_sstk = cpu_hi_desc + 245 offsetof(cpu_desc_table_t, sstk.top); 246 247 /* 248 * LDT descriptors are mapped into a seperate area. 249 */ 250 cdi->cdi_ldt = (struct fake_descriptor *) 251 pmap_cpu_high_shared_remap( 252 cdp->cpu_number, 253 HIGH_CPU_LDT_BEGIN, 254 (vm_offset_t) cdp->cpu_ldtp, 255 HIGH_CPU_LDT_END - HIGH_CPU_LDT_BEGIN + 1); 256 257 /* 258 * Copy the tables 259 */ 260 bcopy((char *)master_idt, 261 (char *)cdt->idt, 262 sizeof(master_idt)); 263 bcopy((char *)master_gdt, 264 (char *)cdt->gdt, 265 sizeof(master_gdt)); 266 bcopy((char *)master_ldt, 267 (char *)cdp->cpu_ldtp, 268 sizeof(master_ldt)); 269 bzero((char *)&cdt->ktss, 270 sizeof(struct i386_tss)); 271 272#if MACH_KDB 273 cdi->cdi_dbtss = (struct i386_tss *) (cpu_hi_desc + 274 offsetof(cpu_desc_table_t, dbtss)); 275 bcopy((char *)&master_dbtss, 276 (char *)&cdt->dbtss, 277 sizeof(struct i386_tss)); 278#endif /* MACH_KDB */ 279 280 /* 281 * Fix up the entries in the GDT to point to 282 * this LDT and this TSS. 283 */ 284 cdt->gdt[sel_idx(KERNEL_LDT)] = ldt_desc_pattern; 285 cdt->gdt[sel_idx(KERNEL_LDT)].offset = (vm_offset_t) cdi->cdi_ldt; 286 fix_desc(&cdt->gdt[sel_idx(KERNEL_LDT)], 1); 287 288 cdt->gdt[sel_idx(USER_LDT)] = ldt_desc_pattern; 289 cdt->gdt[sel_idx(USER_LDT)].offset = (vm_offset_t) cdi->cdi_ldt; 290 fix_desc(&cdt->gdt[sel_idx(USER_LDT)], 1); 291 292 cdt->gdt[sel_idx(KERNEL_TSS)] = tss_desc_pattern; 293 cdt->gdt[sel_idx(KERNEL_TSS)].offset = (vm_offset_t) cdi->cdi_ktss; 294 fix_desc(&cdt->gdt[sel_idx(KERNEL_TSS)], 1); 295 296 cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern; 297 cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp; 298 fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1); 299 300#if MACH_KDB 301 cdt->gdt[sel_idx(DEBUG_TSS)] = tss_desc_pattern; 302 cdt->gdt[sel_idx(DEBUG_TSS)].offset = (vm_offset_t) cdi->cdi_dbtss; 303 fix_desc(&cdt->gdt[sel_idx(DEBUG_TSS)], 1); 304 305 cdt->dbtss.esp0 = (int)(db_task_stack_store + 306 (INTSTACK_SIZE * (cdp->cpu_number)) - sizeof (natural_t)); 307 cdt->dbtss.esp = cdt->dbtss.esp0; 308 cdt->dbtss.eip = (int)&db_task_start; 309#endif /* MACH_KDB */ 310 311 cdt->ktss.ss0 = KERNEL_DS; 312 cdt->ktss.io_bit_map_offset = 0x0FFF; /* no IO bitmap */ 313 314 cpu_userwindow_init(cdp->cpu_number); 315 cpu_physwindow_init(cdp->cpu_number); 316 317 } 318 319} 320 321void 322cpu_desc_init64( 323 cpu_data_t *cdp, 324 boolean_t is_boot_cpu) 325{ 326 cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) 327 cdp->cpu_desc_tablep; 328 cpu_desc_index_t *cdi = &cdp->cpu_desc_index; 329 330 if (is_boot_cpu) { 331 /* 332 * Master CPU uses the tables built at boot time. 333 * Just set the index pointers to the low memory space. 334 * Note that in 64-bit mode these are addressed in the 335 * double-mapped window (uber-space). 336 */ 337 cdi->cdi_ktss = (struct i386_tss *) &master_ktss64; 338 cdi->cdi_sstk = (vm_offset_t) &master_sstk.top; 339 cdi->cdi_gdt = master_gdt; 340 cdi->cdi_idt = (struct fake_descriptor *) &master_idt64; 341 cdi->cdi_ldt = (struct fake_descriptor *) &master_ldt; 342 343 /* Replace the expanded LDT and TSS slots in the GDT: */ 344 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] = 345 kernel_ldt_desc64; 346 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] = 347 kernel_tss_desc64; 348 349 /* 350 * Fix up the expanded descriptors for 64-bit. 351 */ 352 fix_desc64((void *) &master_idt64, IDTSZ); 353 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1); 354 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1); 355 356 /* 357 * Set the double-fault stack as IST1 in the 64-bit TSS 358 */ 359 master_ktss64.ist1 = UBER64(df_task_stack_end); 360 361 } else { 362 /* 363 * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel 364 * heap (cpu_desc_table) and double-mapped in uber-space 365 * (over 4GB). 366 * LDT descriptors are mapped into a separate area. 367 */ 368 cdi->cdi_gdt = (struct fake_descriptor *)cdt->gdt; 369 cdi->cdi_idt = (struct fake_descriptor *)cdt->idt; 370 cdi->cdi_ktss = (struct i386_tss *)&cdt->ktss; 371 cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top; 372 cdi->cdi_ldt = cdp->cpu_ldtp; 373 374 /* 375 * Copy the tables 376 */ 377 bcopy((char *)master_idt64, 378 (char *)cdt->idt, 379 sizeof(master_idt64)); 380 bcopy((char *)master_gdt, 381 (char *)cdt->gdt, 382 sizeof(master_gdt)); 383 bcopy((char *)master_ldt, 384 (char *)cdp->cpu_ldtp, 385 sizeof(master_ldt)); 386 bcopy((char *)&master_ktss64, 387 (char *)&cdt->ktss, 388 sizeof(struct x86_64_tss)); 389 390 /* 391 * Fix up the entries in the GDT to point to 392 * this LDT and this TSS. 393 */ 394 kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt; 395 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] = 396 kernel_ldt_desc64; 397 fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1); 398 399 kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt; 400 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] = 401 kernel_ldt_desc64; 402 fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1); 403 404 kernel_tss_desc64.offset[0] = (vm_offset_t) cdi->cdi_ktss; 405 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] = 406 kernel_tss_desc64; 407 fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1); 408 409 cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern; 410 cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp; 411 fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1); 412 413 /* Set double-fault stack as IST1 */ 414 cdt->ktss.ist1 = UBER64((unsigned long)cdt->dfstk 415 + sizeof(cdt->dfstk)); 416 417 /* 418 * Allocate copyio windows. 419 */ 420 cpu_userwindow_init(cdp->cpu_number); 421 cpu_physwindow_init(cdp->cpu_number); 422 } 423 424 /* Require that the top of the sysenter stack is 16-byte aligned */ 425 if ((cdi->cdi_sstk % 16) != 0) 426 panic("cpu_desc_init64() sysenter stack not 16-byte aligned"); 427} 428 429/* 430 * Set MSRs for sysenter/sysexit for 64-bit. 431 */ 432static void 433fast_syscall_init64(void) 434{ 435 wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); 436 wrmsr64(MSR_IA32_SYSENTER_EIP, UBER64(hi64_sysenter)); 437 wrmsr64(MSR_IA32_SYSENTER_ESP, UBER64(current_sstk())); 438 439 /* Enable syscall/sysret */ 440 wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE); 441 442 /* 443 * MSRs for 64-bit syscall/sysret 444 * Note USER_CS because sysret uses this + 16 when returning to 445 * 64-bit code. 446 */ 447 wrmsr64(MSR_IA32_LSTAR, UBER64(hi64_syscall)); 448 wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) | 449 (((uint64_t)KERNEL64_CS) << 32)); 450 /* 451 * Emulate eflags cleared by sysenter but note that 452 * we also clear the trace trap to avoid the complications 453 * of single-stepping into a syscall. The nested task bit 454 * is also cleared to avoid a spurious "task switch" 455 * should we choose to return via an IRET. 456 */ 457 wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT); 458 459 /* 460 * Set the Kernel GS base MSR to point to per-cpu data in uber-space. 461 * The uber-space handler (hi64_syscall) uses the swapgs instruction. 462 */ 463 wrmsr64(MSR_IA32_KERNEL_GS_BASE, 464 UBER64((unsigned long)current_cpu_datap())); 465 466#if ONLY_SAFE_FOR_LINDA_SERIAL 467 kprintf("fast_syscall_init64() KERNEL_GS_BASE=0x%016llx\n", 468 rdmsr64(MSR_IA32_KERNEL_GS_BASE)); 469#endif 470} 471 472/* 473 * Set MSRs for sysenter/sysexit 474 */ 475static void 476fast_syscall_init(void) 477{ 478 wrmsr(MSR_IA32_SYSENTER_CS, SYSENTER_CS, 0); 479 wrmsr(MSR_IA32_SYSENTER_EIP, HI_TEXT(hi_sysenter), 0); 480 wrmsr(MSR_IA32_SYSENTER_ESP, current_sstk(), 0); 481} 482 483cpu_data_t * 484cpu_data_alloc(boolean_t is_boot_cpu) 485{ 486 int ret; 487 cpu_data_t *cdp; 488 489 if (is_boot_cpu) { 490 assert(real_ncpus == 1); 491 cdp = &cpu_data_master; 492 if (cdp->cpu_processor == NULL) { 493 simple_lock_init(&cpu_lock, 0); 494 cdp->cpu_processor = cpu_processor_alloc(TRUE); 495 cdp->cpu_pmap = pmap_cpu_alloc(TRUE); 496 cpu_desc_init(cdp, TRUE); 497 fast_syscall_init(); 498 queue_init(&cdp->rtclock_timer.queue); 499 cdp->rtclock_timer.deadline = EndOfAllTime; 500 } 501 return cdp; 502 } 503 504 /* Check count before making allocations */ 505 if (real_ncpus >= max_ncpus) 506 return NULL; 507 508 /* 509 * Allocate per-cpu data: 510 */ 511 ret = kmem_alloc(kernel_map, 512 (vm_offset_t *) &cdp, sizeof(cpu_data_t)); 513 if (ret != KERN_SUCCESS) { 514 printf("cpu_data_alloc() failed, ret=%d\n", ret); 515 goto abort; 516 } 517 bzero((void*) cdp, sizeof(cpu_data_t)); 518 cdp->cpu_this = cdp; 519 520 /* Propagate mode */ 521 cdp->cpu_is64bit = cpu_mode_is64bit(); 522 523 /* 524 * Allocate interrupt stack: 525 */ 526 ret = kmem_alloc(kernel_map, 527 (vm_offset_t *) &cdp->cpu_int_stack_top, 528 INTSTACK_SIZE); 529 if (ret != KERN_SUCCESS) { 530 printf("cpu_data_alloc() int stack failed, ret=%d\n", ret); 531 goto abort; 532 } 533 bzero((void*) cdp->cpu_int_stack_top, INTSTACK_SIZE); 534 cdp->cpu_int_stack_top += INTSTACK_SIZE; 535 536 /* 537 * Allocate descriptor table: 538 * Size depends on cpu mode. 539 */ 540 ret = kmem_alloc(kernel_map, 541 (vm_offset_t *) &cdp->cpu_desc_tablep, 542 cdp->cpu_is64bit ? sizeof(cpu_desc_table64_t) 543 : sizeof(cpu_desc_table_t)); 544 if (ret != KERN_SUCCESS) { 545 printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret); 546 goto abort; 547 } 548 549 /* 550 * Allocate LDT 551 */ 552 ret = kmem_alloc(kernel_map, 553 (vm_offset_t *) &cdp->cpu_ldtp, 554 sizeof(struct real_descriptor) * LDTSZ); 555 if (ret != KERN_SUCCESS) { 556 printf("cpu_data_alloc() ldt failed, ret=%d\n", ret); 557 goto abort; 558 } 559 560 /* Machine-check shadow register allocation. */ 561 mca_cpu_alloc(cdp); 562 563 simple_lock(&cpu_lock); 564 if (real_ncpus >= max_ncpus) { 565 simple_unlock(&cpu_lock); 566 goto abort; 567 } 568 cpu_data_ptr[real_ncpus] = cdp; 569 cdp->cpu_number = real_ncpus; 570 real_ncpus++; 571 simple_unlock(&cpu_lock); 572 573 cdp->cpu_nanotime = &rtc_nanotime_info; 574 queue_init(&cdp->rtclock_timer.queue); 575 cdp->rtclock_timer.deadline = EndOfAllTime; 576 577 kprintf("cpu_data_alloc(%d) %p desc_table: %p " 578 "ldt: %p " 579 "int_stack: 0x%x-0x%x\n", 580 cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp, 581 cdp->cpu_int_stack_top - INTSTACK_SIZE, cdp->cpu_int_stack_top); 582 583 return cdp; 584 585abort: 586 if (cdp) { 587 if (cdp->cpu_desc_tablep) 588 kfree((void *) cdp->cpu_desc_tablep, 589 sizeof(*cdp->cpu_desc_tablep)); 590 if (cdp->cpu_int_stack_top) 591 kfree((void *) (cdp->cpu_int_stack_top - INTSTACK_SIZE), 592 INTSTACK_SIZE); 593 kfree((void *) cdp, sizeof(*cdp)); 594 } 595 return NULL; 596} 597 598boolean_t 599valid_user_segment_selectors(uint16_t cs, 600 uint16_t ss, 601 uint16_t ds, 602 uint16_t es, 603 uint16_t fs, 604 uint16_t gs) 605{ 606 return valid_user_code_selector(cs) && 607 valid_user_stack_selector(ss) && 608 valid_user_data_selector(ds) && 609 valid_user_data_selector(es) && 610 valid_user_data_selector(fs) && 611 valid_user_data_selector(gs); 612} 613 614 615static vm_offset_t user_window_base = 0; 616 617void 618cpu_userwindow_init(int cpu) 619{ 620 cpu_data_t *cdp = cpu_data_ptr[cpu]; 621 cpu_desc_index_t *cdi = &cdp->cpu_desc_index; 622 vm_offset_t user_window; 623 vm_offset_t vaddr; 624 int num_cpus; 625 626 num_cpus = ml_get_max_cpus(); 627 628 if (cpu >= num_cpus) 629 panic("cpu_userwindow_init: cpu > num_cpus"); 630 631 if (user_window_base == 0) { 632 633 if (vm_allocate(kernel_map, &vaddr, 634 (NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE, 635 VM_FLAGS_ANYWHERE) != KERN_SUCCESS) 636 panic("cpu_userwindow_init: " 637 "couldn't allocate user map window"); 638 639 /* 640 * window must start on a page table boundary 641 * in the virtual address space 642 */ 643 user_window_base = (vaddr + (NBPDE - 1)) & ~(NBPDE - 1); 644 645 /* 646 * get rid of any allocation leading up to our 647 * starting boundary 648 */ 649 vm_deallocate(kernel_map, vaddr, user_window_base - vaddr); 650 651 /* 652 * get rid of tail that we don't need 653 */ 654 user_window = user_window_base + 655 (NBPDE * NCOPY_WINDOWS * num_cpus); 656 657 vm_deallocate(kernel_map, user_window, 658 (vaddr + 659 ((NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE)) - 660 user_window); 661 } 662 663 user_window = user_window_base + (cpu * NCOPY_WINDOWS * NBPDE); 664 665 cdp->cpu_copywindow_base = user_window; 666 cdp->cpu_copywindow_pdp = pmap_pde(kernel_pmap, user_window); 667 668 cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)] = userwindow_desc_pattern; 669 cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)].offset = user_window; 670 671 fix_desc(&cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)], 1); 672 673} 674 675void 676cpu_physwindow_init(int cpu) 677{ 678 cpu_data_t *cdp = cpu_data_ptr[cpu]; 679 cpu_desc_index_t *cdi = &cdp->cpu_desc_index; 680 vm_offset_t phys_window = cdp->cpu_physwindow_base; 681 682 if (phys_window == 0) { 683 if (vm_allocate(kernel_map, &phys_window, 684 PAGE_SIZE, VM_FLAGS_ANYWHERE) 685 != KERN_SUCCESS) 686 panic("cpu_physwindow_init: " 687 "couldn't allocate phys map window"); 688 689 /* 690 * make sure the page that encompasses the 691 * pte pointer we're interested in actually 692 * exists in the page table 693 */ 694 pmap_expand(kernel_pmap, phys_window); 695 696 cdp->cpu_physwindow_base = phys_window; 697 cdp->cpu_physwindow_ptep = vtopte(phys_window); 698 } 699 700 cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)] = physwindow_desc_pattern; 701 cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)].offset = phys_window; 702 703 fix_desc(&cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)], 1); 704} 705 706 707typedef struct { 708 uint16_t length; 709 uint32_t offset[2]; 710} __attribute__((__packed__)) table_descriptor64_t; 711 712extern table_descriptor64_t gdtptr64; 713extern table_descriptor64_t idtptr64; 714/* 715 * Load the segment descriptor tables for the current processor. 716 */ 717void 718cpu_desc_load64(cpu_data_t *cdp) 719{ 720 cpu_desc_index_t *cdi = &cdp->cpu_desc_index; 721 722 /* 723 * Load up the new descriptors etc 724 * ml_load_desc64() expects these global pseudo-descriptors: 725 * gdtptr64 -> master_gdt 726 * idtptr64 -> master_idt64 727 * These are 10-byte descriptors with 64-bit addresses into 728 * uber-space. 729 */ 730 gdtptr64.length = sizeof(master_gdt) - 1; 731 gdtptr64.offset[0] = (uint32_t) cdi->cdi_gdt; 732 gdtptr64.offset[1] = KERNEL_UBER_BASE_HI32; 733 idtptr64.length = sizeof(master_idt64) - 1; 734 idtptr64.offset[0] = (uint32_t) cdi->cdi_idt; 735 idtptr64.offset[1] = KERNEL_UBER_BASE_HI32; 736 737 /* Make sure busy bit is cleared in the TSS */ 738 gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY; 739 740 ml_load_desc64(); 741 742#if ONLY_SAFE_FOR_LINDA_SERIAL 743 kprintf("64-bit descriptor tables loaded\n"); 744#endif 745} 746 747void 748cpu_mode_init(cpu_data_t *cdp) 749{ 750 if (cpu_mode_is64bit()) { 751 cpu_IA32e_enable(cdp); 752 cpu_desc_load64(cdp); 753 fast_syscall_init64(); 754 } else { 755 fast_syscall_init(); 756 } 757 758 /* Call for per-cpu pmap mode initialization */ 759 pmap_cpu_init(); 760 761} 762 763