npx.c revision 273989
1/*- 2 * Copyright (c) 1990 William Jolitz. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD: head/sys/i386/isa/npx.c 273989 2014-11-02 21:34:24Z jhb $"); 35 36#include "opt_cpu.h" 37#include "opt_isa.h" 38#include "opt_npx.h" 39 40#include <sys/param.h> 41#include <sys/systm.h> 42#include <sys/bus.h> 43#include <sys/kernel.h> 44#include <sys/lock.h> 45#include <sys/malloc.h> 46#include <sys/module.h> 47#include <sys/mutex.h> 48#include <sys/mutex.h> 49#include <sys/proc.h> 50#include <sys/smp.h> 51#include <sys/sysctl.h> 52#include <machine/bus.h> 53#include <sys/rman.h> 54#ifdef NPX_DEBUG 55#include <sys/syslog.h> 56#endif 57#include <sys/signalvar.h> 58#include <vm/uma.h> 59 60#include <machine/asmacros.h> 61#include <machine/cputypes.h> 62#include <machine/frame.h> 63#include <machine/md_var.h> 64#include <machine/pcb.h> 65#include <machine/psl.h> 66#include <machine/resource.h> 67#include <machine/specialreg.h> 68#include <machine/segments.h> 69#include <machine/ucontext.h> 70 71#include <machine/intr_machdep.h> 72#ifdef XEN 73#include <xen/xen-os.h> 74#include <xen/hypervisor.h> 75#endif 76 77#ifdef DEV_ISA 78#include <isa/isavar.h> 79#endif 80 81#if !defined(CPU_DISABLE_SSE) && defined(I686_CPU) 82#define CPU_ENABLE_SSE 83#endif 84 85/* 86 * 387 and 287 Numeric Coprocessor Extension (NPX) Driver. 87 */ 88 89#if defined(__GNUCLIKE_ASM) && !defined(lint) 90 91#define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) 92#define fnclex() __asm __volatile("fnclex") 93#define fninit() __asm __volatile("fninit") 94#define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr))) 95#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) 96#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) 97#define fp_divide_by_0() __asm __volatile( \ 98 "fldz; fld1; fdiv %st,%st(1); fnop") 99#define frstor(addr) __asm __volatile("frstor %0" : : "m" (*(addr))) 100#ifdef CPU_ENABLE_SSE 101#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) 102#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) 103#define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) 104#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) 105#endif 106#else /* !(__GNUCLIKE_ASM && !lint) */ 107 108void fldcw(u_short cw); 109void fnclex(void); 110void fninit(void); 111void fnsave(caddr_t addr); 112void fnstcw(caddr_t addr); 113void fnstsw(caddr_t addr); 114void fp_divide_by_0(void); 115void frstor(caddr_t addr); 116#ifdef CPU_ENABLE_SSE 117void fxsave(caddr_t addr); 118void fxrstor(caddr_t addr); 119void ldmxcsr(u_int csr); 120void stmxcsr(u_int *csr); 121#endif 122 123#endif /* __GNUCLIKE_ASM && !lint */ 124 125#ifdef XEN 126#define start_emulating() (HYPERVISOR_fpu_taskswitch(1)) 127#define stop_emulating() (HYPERVISOR_fpu_taskswitch(0)) 128#else 129#define start_emulating() load_cr0(rcr0() | CR0_TS) 130#define stop_emulating() clts() 131#endif 132 133#ifdef CPU_ENABLE_SSE 134#define GET_FPU_CW(thread) \ 135 (cpu_fxsr ? \ 136 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \ 137 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw) 138#define GET_FPU_SW(thread) \ 139 (cpu_fxsr ? \ 140 (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \ 141 (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw) 142#define SET_FPU_CW(savefpu, value) do { \ 143 if (cpu_fxsr) \ 144 (savefpu)->sv_xmm.sv_env.en_cw = (value); \ 145 else \ 146 (savefpu)->sv_87.sv_env.en_cw = (value); \ 147} while (0) 148#else /* CPU_ENABLE_SSE */ 149#define GET_FPU_CW(thread) \ 150 (thread->td_pcb->pcb_save->sv_87.sv_env.en_cw) 151#define GET_FPU_SW(thread) \ 152 (thread->td_pcb->pcb_save->sv_87.sv_env.en_sw) 153#define SET_FPU_CW(savefpu, value) \ 154 (savefpu)->sv_87.sv_env.en_cw = (value) 155#endif /* CPU_ENABLE_SSE */ 156 157typedef u_char bool_t; 158 159#ifdef CPU_ENABLE_SSE 160static void fpu_clean_state(void); 161#endif 162 163static void fpusave(union savefpu *); 164static void fpurstor(union savefpu *); 165static int npx_attach(device_t dev); 166static void npx_identify(driver_t *driver, device_t parent); 167static int npx_probe(device_t dev); 168 169int hw_float; 170 171SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, 172 &hw_float, 0, "Floating point instructions executed in hardware"); 173 174static volatile u_int npx_traps_while_probing; 175static union savefpu npx_initialstate; 176 177alias_for_inthand_t probetrap; 178__asm(" \n\ 179 .text \n\ 180 .p2align 2,0x90 \n\ 181 .type " __XSTRING(CNAME(probetrap)) ",@function \n\ 182" __XSTRING(CNAME(probetrap)) ": \n\ 183 ss \n\ 184 incl " __XSTRING(CNAME(npx_traps_while_probing)) " \n\ 185 fnclex \n\ 186 iret \n\ 187"); 188 189/* 190 * Identify routine. Create a connection point on our parent for probing. 191 */ 192static void 193npx_identify(driver, parent) 194 driver_t *driver; 195 device_t parent; 196{ 197 device_t child; 198 199 child = BUS_ADD_CHILD(parent, 0, "npx", 0); 200 if (child == NULL) 201 panic("npx_identify"); 202} 203 204/* 205 * Probe routine. Set flags to tell npxattach() what to do. Set up an 206 * interrupt handler if npx needs to use interrupts. 207 */ 208static int 209npx_probe(device_t dev) 210{ 211 struct gate_descriptor save_idt_npxtrap; 212 u_short control, status; 213 214 device_set_desc(dev, "math processor"); 215 216 /* 217 * Modern CPUs all have an FPU that uses the INT16 interface 218 * and provide a simple way to verify that, so handle the 219 * common case right away. 220 */ 221 if (cpu_feature & CPUID_FPU) { 222 hw_float = 1; 223 device_quiet(dev); 224 return (0); 225 } 226 227 save_idt_npxtrap = idt[IDT_MF]; 228 setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL, 229 GSEL(GCODE_SEL, SEL_KPL)); 230 231 /* 232 * Don't trap while we're probing. 233 */ 234 stop_emulating(); 235 236 /* 237 * Finish resetting the coprocessor, if any. If there is an error 238 * pending, then we may get a bogus IRQ13, but npx_intr() will handle 239 * it OK. Bogus halts have never been observed, but we enabled 240 * IRQ13 and cleared the BUSY# latch early to handle them anyway. 241 */ 242 fninit(); 243 244 /* 245 * Don't use fwait here because it might hang. 246 * Don't use fnop here because it usually hangs if there is no FPU. 247 */ 248 DELAY(1000); /* wait for any IRQ13 */ 249#ifdef DIAGNOSTIC 250 if (npx_traps_while_probing != 0) 251 printf("fninit caused %u bogus npx trap(s)\n", 252 npx_traps_while_probing); 253#endif 254 /* 255 * Check for a status of mostly zero. 256 */ 257 status = 0x5a5a; 258 fnstsw(&status); 259 if ((status & 0xb8ff) == 0) { 260 /* 261 * Good, now check for a proper control word. 262 */ 263 control = 0x5a5a; 264 fnstcw(&control); 265 if ((control & 0x1f3f) == 0x033f) { 266 /* 267 * We have an npx, now divide by 0 to see if exception 268 * 16 works. 269 */ 270 control &= ~(1 << 2); /* enable divide by 0 trap */ 271 fldcw(control); 272#ifdef FPU_ERROR_BROKEN 273 /* 274 * FPU error signal doesn't work on some CPU 275 * accelerator board. 276 */ 277 hw_float = 1; 278 return (0); 279#endif 280 npx_traps_while_probing = 0; 281 fp_divide_by_0(); 282 if (npx_traps_while_probing != 0) { 283 /* 284 * Good, exception 16 works. 285 */ 286 hw_float = 1; 287 goto cleanup; 288 } 289 device_printf(dev, 290 "FPU does not use exception 16 for error reporting\n"); 291 goto cleanup; 292 } 293 } 294 295 /* 296 * Probe failed. Floating point simply won't work. 297 * Notify user and disable FPU/MMX/SSE instruction execution. 298 */ 299 device_printf(dev, "WARNING: no FPU!\n"); 300 __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : 301 "n" (CR0_EM | CR0_MP) : "ax"); 302 303cleanup: 304 idt[IDT_MF] = save_idt_npxtrap; 305 return (hw_float ? 0 : ENXIO); 306} 307 308/* 309 * Attach routine - announce which it is, and wire into system 310 */ 311static int 312npx_attach(device_t dev) 313{ 314 315 npxinit(); 316 critical_enter(); 317 stop_emulating(); 318 fpusave(&npx_initialstate); 319 start_emulating(); 320#ifdef CPU_ENABLE_SSE 321 if (cpu_fxsr) { 322 if (npx_initialstate.sv_xmm.sv_env.en_mxcsr_mask) 323 cpu_mxcsr_mask = 324 npx_initialstate.sv_xmm.sv_env.en_mxcsr_mask; 325 else 326 cpu_mxcsr_mask = 0xFFBF; 327 bzero(npx_initialstate.sv_xmm.sv_fp, 328 sizeof(npx_initialstate.sv_xmm.sv_fp)); 329 bzero(npx_initialstate.sv_xmm.sv_xmm, 330 sizeof(npx_initialstate.sv_xmm.sv_xmm)); 331 /* XXX might need even more zeroing. */ 332 } else 333#endif 334 bzero(npx_initialstate.sv_87.sv_ac, 335 sizeof(npx_initialstate.sv_87.sv_ac)); 336 critical_exit(); 337 338 return (0); 339} 340 341/* 342 * Initialize floating point unit. 343 */ 344void 345npxinit(void) 346{ 347 static union savefpu dummy; 348 register_t saveintr; 349 u_short control; 350 351 if (!hw_float) 352 return; 353 /* 354 * fninit has the same h/w bugs as fnsave. Use the detoxified 355 * fnsave to throw away any junk in the fpu. npxsave() initializes 356 * the fpu and sets fpcurthread = NULL as important side effects. 357 * 358 * It is too early for critical_enter() to work on AP. 359 */ 360 saveintr = intr_disable(); 361 npxsave(&dummy); 362 stop_emulating(); 363#ifdef CPU_ENABLE_SSE 364 /* XXX npxsave() doesn't actually initialize the fpu in the SSE case. */ 365 if (cpu_fxsr) 366 fninit(); 367#endif 368 control = __INITIAL_NPXCW__; 369 fldcw(control); 370#ifdef CPU_ENABLE_SSE 371 if (cpu_fxsr) { 372 mxcsr = __INITIAL_MXCSR__; 373 ldmxcsr(mxcsr); 374 } 375#endif 376 start_emulating(); 377 intr_restore(saveintr); 378} 379 380/* 381 * Free coprocessor (if we have it). 382 */ 383void 384npxexit(td) 385 struct thread *td; 386{ 387 388 critical_enter(); 389 if (curthread == PCPU_GET(fpcurthread)) 390 npxsave(curpcb->pcb_save); 391 critical_exit(); 392#ifdef NPX_DEBUG 393 if (hw_float) { 394 u_int masked_exceptions; 395 396 masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f; 397 /* 398 * Log exceptions that would have trapped with the old 399 * control word (overflow, divide by 0, and invalid operand). 400 */ 401 if (masked_exceptions & 0x0d) 402 log(LOG_ERR, 403 "pid %d (%s) exited with masked floating point exceptions 0x%02x\n", 404 td->td_proc->p_pid, td->td_proc->p_comm, 405 masked_exceptions); 406 } 407#endif 408} 409 410int 411npxformat() 412{ 413 414 if (!hw_float) 415 return (_MC_FPFMT_NODEV); 416#ifdef CPU_ENABLE_SSE 417 if (cpu_fxsr) 418 return (_MC_FPFMT_XMM); 419#endif 420 return (_MC_FPFMT_387); 421} 422 423/* 424 * The following mechanism is used to ensure that the FPE_... value 425 * that is passed as a trapcode to the signal handler of the user 426 * process does not have more than one bit set. 427 * 428 * Multiple bits may be set if the user process modifies the control 429 * word while a status word bit is already set. While this is a sign 430 * of bad coding, we have no choise than to narrow them down to one 431 * bit, since we must not send a trapcode that is not exactly one of 432 * the FPE_ macros. 433 * 434 * The mechanism has a static table with 127 entries. Each combination 435 * of the 7 FPU status word exception bits directly translates to a 436 * position in this table, where a single FPE_... value is stored. 437 * This FPE_... value stored there is considered the "most important" 438 * of the exception bits and will be sent as the signal code. The 439 * precedence of the bits is based upon Intel Document "Numerical 440 * Applications", Chapter "Special Computational Situations". 441 * 442 * The macro to choose one of these values does these steps: 1) Throw 443 * away status word bits that cannot be masked. 2) Throw away the bits 444 * currently masked in the control word, assuming the user isn't 445 * interested in them anymore. 3) Reinsert status word bit 7 (stack 446 * fault) if it is set, which cannot be masked but must be presered. 447 * 4) Use the remaining bits to point into the trapcode table. 448 * 449 * The 6 maskable bits in order of their preference, as stated in the 450 * above referenced Intel manual: 451 * 1 Invalid operation (FP_X_INV) 452 * 1a Stack underflow 453 * 1b Stack overflow 454 * 1c Operand of unsupported format 455 * 1d SNaN operand. 456 * 2 QNaN operand (not an exception, irrelavant here) 457 * 3 Any other invalid-operation not mentioned above or zero divide 458 * (FP_X_INV, FP_X_DZ) 459 * 4 Denormal operand (FP_X_DNML) 460 * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) 461 * 6 Inexact result (FP_X_IMP) 462 */ 463static char fpetable[128] = { 464 0, 465 FPE_FLTINV, /* 1 - INV */ 466 FPE_FLTUND, /* 2 - DNML */ 467 FPE_FLTINV, /* 3 - INV | DNML */ 468 FPE_FLTDIV, /* 4 - DZ */ 469 FPE_FLTINV, /* 5 - INV | DZ */ 470 FPE_FLTDIV, /* 6 - DNML | DZ */ 471 FPE_FLTINV, /* 7 - INV | DNML | DZ */ 472 FPE_FLTOVF, /* 8 - OFL */ 473 FPE_FLTINV, /* 9 - INV | OFL */ 474 FPE_FLTUND, /* A - DNML | OFL */ 475 FPE_FLTINV, /* B - INV | DNML | OFL */ 476 FPE_FLTDIV, /* C - DZ | OFL */ 477 FPE_FLTINV, /* D - INV | DZ | OFL */ 478 FPE_FLTDIV, /* E - DNML | DZ | OFL */ 479 FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ 480 FPE_FLTUND, /* 10 - UFL */ 481 FPE_FLTINV, /* 11 - INV | UFL */ 482 FPE_FLTUND, /* 12 - DNML | UFL */ 483 FPE_FLTINV, /* 13 - INV | DNML | UFL */ 484 FPE_FLTDIV, /* 14 - DZ | UFL */ 485 FPE_FLTINV, /* 15 - INV | DZ | UFL */ 486 FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ 487 FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ 488 FPE_FLTOVF, /* 18 - OFL | UFL */ 489 FPE_FLTINV, /* 19 - INV | OFL | UFL */ 490 FPE_FLTUND, /* 1A - DNML | OFL | UFL */ 491 FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ 492 FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ 493 FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ 494 FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ 495 FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ 496 FPE_FLTRES, /* 20 - IMP */ 497 FPE_FLTINV, /* 21 - INV | IMP */ 498 FPE_FLTUND, /* 22 - DNML | IMP */ 499 FPE_FLTINV, /* 23 - INV | DNML | IMP */ 500 FPE_FLTDIV, /* 24 - DZ | IMP */ 501 FPE_FLTINV, /* 25 - INV | DZ | IMP */ 502 FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ 503 FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ 504 FPE_FLTOVF, /* 28 - OFL | IMP */ 505 FPE_FLTINV, /* 29 - INV | OFL | IMP */ 506 FPE_FLTUND, /* 2A - DNML | OFL | IMP */ 507 FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ 508 FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ 509 FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ 510 FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ 511 FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ 512 FPE_FLTUND, /* 30 - UFL | IMP */ 513 FPE_FLTINV, /* 31 - INV | UFL | IMP */ 514 FPE_FLTUND, /* 32 - DNML | UFL | IMP */ 515 FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ 516 FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ 517 FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ 518 FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ 519 FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ 520 FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ 521 FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ 522 FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ 523 FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ 524 FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ 525 FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ 526 FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ 527 FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ 528 FPE_FLTSUB, /* 40 - STK */ 529 FPE_FLTSUB, /* 41 - INV | STK */ 530 FPE_FLTUND, /* 42 - DNML | STK */ 531 FPE_FLTSUB, /* 43 - INV | DNML | STK */ 532 FPE_FLTDIV, /* 44 - DZ | STK */ 533 FPE_FLTSUB, /* 45 - INV | DZ | STK */ 534 FPE_FLTDIV, /* 46 - DNML | DZ | STK */ 535 FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ 536 FPE_FLTOVF, /* 48 - OFL | STK */ 537 FPE_FLTSUB, /* 49 - INV | OFL | STK */ 538 FPE_FLTUND, /* 4A - DNML | OFL | STK */ 539 FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ 540 FPE_FLTDIV, /* 4C - DZ | OFL | STK */ 541 FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ 542 FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ 543 FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ 544 FPE_FLTUND, /* 50 - UFL | STK */ 545 FPE_FLTSUB, /* 51 - INV | UFL | STK */ 546 FPE_FLTUND, /* 52 - DNML | UFL | STK */ 547 FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ 548 FPE_FLTDIV, /* 54 - DZ | UFL | STK */ 549 FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ 550 FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ 551 FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ 552 FPE_FLTOVF, /* 58 - OFL | UFL | STK */ 553 FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ 554 FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ 555 FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ 556 FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ 557 FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ 558 FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ 559 FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ 560 FPE_FLTRES, /* 60 - IMP | STK */ 561 FPE_FLTSUB, /* 61 - INV | IMP | STK */ 562 FPE_FLTUND, /* 62 - DNML | IMP | STK */ 563 FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ 564 FPE_FLTDIV, /* 64 - DZ | IMP | STK */ 565 FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ 566 FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ 567 FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ 568 FPE_FLTOVF, /* 68 - OFL | IMP | STK */ 569 FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ 570 FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ 571 FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ 572 FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ 573 FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ 574 FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ 575 FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ 576 FPE_FLTUND, /* 70 - UFL | IMP | STK */ 577 FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ 578 FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ 579 FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ 580 FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ 581 FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ 582 FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ 583 FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ 584 FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ 585 FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ 586 FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ 587 FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ 588 FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ 589 FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ 590 FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ 591 FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ 592}; 593 594/* 595 * Read the FP status and control words, then generate si_code value 596 * for SIGFPE. The error code chosen will be one of the 597 * FPE_... macros. It will be sent as the second argument to old 598 * BSD-style signal handlers and as "siginfo_t->si_code" (second 599 * argument) to SA_SIGINFO signal handlers. 600 * 601 * Some time ago, we cleared the x87 exceptions with FNCLEX there. 602 * Clearing exceptions was necessary mainly to avoid IRQ13 bugs. The 603 * usermode code which understands the FPU hardware enough to enable 604 * the exceptions, can also handle clearing the exception state in the 605 * handler. The only consequence of not clearing the exception is the 606 * rethrow of the SIGFPE on return from the signal handler and 607 * reexecution of the corresponding instruction. 608 * 609 * For XMM traps, the exceptions were never cleared. 610 */ 611int 612npxtrap_x87(void) 613{ 614 u_short control, status; 615 616 if (!hw_float) { 617 printf( 618 "npxtrap_x87: fpcurthread = %p, curthread = %p, hw_float = %d\n", 619 PCPU_GET(fpcurthread), curthread, hw_float); 620 panic("npxtrap from nowhere"); 621 } 622 critical_enter(); 623 624 /* 625 * Interrupt handling (for another interrupt) may have pushed the 626 * state to memory. Fetch the relevant parts of the state from 627 * wherever they are. 628 */ 629 if (PCPU_GET(fpcurthread) != curthread) { 630 control = GET_FPU_CW(curthread); 631 status = GET_FPU_SW(curthread); 632 } else { 633 fnstcw(&control); 634 fnstsw(&status); 635 } 636 critical_exit(); 637 return (fpetable[status & ((~control & 0x3f) | 0x40)]); 638} 639 640#ifdef CPU_ENABLE_SSE 641int 642npxtrap_sse(void) 643{ 644 u_int mxcsr; 645 646 if (!hw_float) { 647 printf( 648 "npxtrap_sse: fpcurthread = %p, curthread = %p, hw_float = %d\n", 649 PCPU_GET(fpcurthread), curthread, hw_float); 650 panic("npxtrap from nowhere"); 651 } 652 critical_enter(); 653 if (PCPU_GET(fpcurthread) != curthread) 654 mxcsr = curthread->td_pcb->pcb_save->sv_xmm.sv_env.en_mxcsr; 655 else 656 stmxcsr(&mxcsr); 657 critical_exit(); 658 return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]); 659} 660#endif 661 662/* 663 * Implement device not available (DNA) exception 664 * 665 * It would be better to switch FP context here (if curthread != fpcurthread) 666 * and not necessarily for every context switch, but it is too hard to 667 * access foreign pcb's. 668 */ 669 670static int err_count = 0; 671 672int 673npxdna(void) 674{ 675 676 if (!hw_float) 677 return (0); 678 critical_enter(); 679 if (PCPU_GET(fpcurthread) == curthread) { 680 printf("npxdna: fpcurthread == curthread %d times\n", 681 ++err_count); 682 stop_emulating(); 683 critical_exit(); 684 return (1); 685 } 686 if (PCPU_GET(fpcurthread) != NULL) { 687 printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n", 688 PCPU_GET(fpcurthread), 689 PCPU_GET(fpcurthread)->td_proc->p_pid, 690 curthread, curthread->td_proc->p_pid); 691 panic("npxdna"); 692 } 693 stop_emulating(); 694 /* 695 * Record new context early in case frstor causes an IRQ13. 696 */ 697 PCPU_SET(fpcurthread, curthread); 698 699#ifdef CPU_ENABLE_SSE 700 if (cpu_fxsr) 701 fpu_clean_state(); 702#endif 703 704 if ((curpcb->pcb_flags & PCB_NPXINITDONE) == 0) { 705 /* 706 * This is the first time this thread has used the FPU or 707 * the PCB doesn't contain a clean FPU state. Explicitly 708 * load an initial state. 709 */ 710 fpurstor(&npx_initialstate); 711 if (curpcb->pcb_initial_npxcw != __INITIAL_NPXCW__) 712 fldcw(curpcb->pcb_initial_npxcw); 713 curpcb->pcb_flags |= PCB_NPXINITDONE; 714 if (PCB_USER_FPU(curpcb)) 715 curpcb->pcb_flags |= PCB_NPXUSERINITDONE; 716 } else { 717 /* 718 * The following fpurstor() may cause an IRQ13 when the 719 * state being restored has a pending error. The error will 720 * appear to have been triggered by the current (npx) user 721 * instruction even when that instruction is a no-wait 722 * instruction that should not trigger an error (e.g., 723 * fnclex). On at least one 486 system all of the no-wait 724 * instructions are broken the same as frstor, so our 725 * treatment does not amplify the breakage. On at least 726 * one 386/Cyrix 387 system, fnclex works correctly while 727 * frstor and fnsave are broken, so our treatment breaks 728 * fnclex if it is the first FPU instruction after a context 729 * switch. 730 */ 731 fpurstor(curpcb->pcb_save); 732 } 733 critical_exit(); 734 735 return (1); 736} 737 738/* 739 * Wrapper for fnsave instruction, partly to handle hardware bugs. When npx 740 * exceptions are reported via IRQ13, spurious IRQ13's may be triggered by 741 * no-wait npx instructions. See the Intel application note AP-578 for 742 * details. This doesn't cause any additional complications here. IRQ13's 743 * are inherently asynchronous unless the CPU is frozen to deliver them -- 744 * one that started in userland may be delivered many instructions later, 745 * after the process has entered the kernel. It may even be delivered after 746 * the fnsave here completes. A spurious IRQ13 for the fnsave is handled in 747 * the same way as a very-late-arriving non-spurious IRQ13 from user mode: 748 * it is normally ignored at first because we set fpcurthread to NULL; it is 749 * normally retriggered in npxdna() after return to user mode. 750 * 751 * npxsave() must be called with interrupts disabled, so that it clears 752 * fpcurthread atomically with saving the state. We require callers to do the 753 * disabling, since most callers need to disable interrupts anyway to call 754 * npxsave() atomically with checking fpcurthread. 755 * 756 * A previous version of npxsave() went to great lengths to excecute fnsave 757 * with interrupts enabled in case executing it froze the CPU. This case 758 * can't happen, at least for Intel CPU/NPX's. Spurious IRQ13's don't imply 759 * spurious freezes. 760 */ 761void 762npxsave(addr) 763 union savefpu *addr; 764{ 765 766 stop_emulating(); 767 fpusave(addr); 768 769 start_emulating(); 770 PCPU_SET(fpcurthread, NULL); 771} 772 773/* 774 * Unconditionally save the current co-processor state across suspend and 775 * resume. 776 */ 777void 778npxsuspend(union savefpu *addr) 779{ 780 register_t cr0; 781 782 if (!hw_float) 783 return; 784 if (PCPU_GET(fpcurthread) == NULL) { 785 *addr = npx_initialstate; 786 return; 787 } 788 cr0 = rcr0(); 789 clts(); 790 fpusave(addr); 791 load_cr0(cr0); 792} 793 794void 795npxresume(union savefpu *addr) 796{ 797 register_t cr0; 798 799 if (!hw_float) 800 return; 801 802 cr0 = rcr0(); 803 clts(); 804 npxinit(); 805 stop_emulating(); 806 fpurstor(addr); 807 load_cr0(cr0); 808} 809 810void 811npxdrop() 812{ 813 struct thread *td; 814 815 /* 816 * Discard pending exceptions in the !cpu_fxsr case so that unmasked 817 * ones don't cause a panic on the next frstor. 818 */ 819#ifdef CPU_ENABLE_SSE 820 if (!cpu_fxsr) 821#endif 822 fnclex(); 823 824 td = PCPU_GET(fpcurthread); 825 KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); 826 CRITICAL_ASSERT(td); 827 PCPU_SET(fpcurthread, NULL); 828 td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; 829 start_emulating(); 830} 831 832/* 833 * Get the user state of the FPU into pcb->pcb_user_save without 834 * dropping ownership (if possible). It returns the FPU ownership 835 * status. 836 */ 837int 838npxgetregs(struct thread *td) 839{ 840 struct pcb *pcb; 841 842 if (!hw_float) 843 return (_MC_FPOWNED_NONE); 844 845 pcb = td->td_pcb; 846 if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) { 847 bcopy(&npx_initialstate, &pcb->pcb_user_save, 848 sizeof(npx_initialstate)); 849 SET_FPU_CW(&pcb->pcb_user_save, pcb->pcb_initial_npxcw); 850 npxuserinited(td); 851 return (_MC_FPOWNED_PCB); 852 } 853 critical_enter(); 854 if (td == PCPU_GET(fpcurthread)) { 855 fpusave(&pcb->pcb_user_save); 856#ifdef CPU_ENABLE_SSE 857 if (!cpu_fxsr) 858#endif 859 /* 860 * fnsave initializes the FPU and destroys whatever 861 * context it contains. Make sure the FPU owner 862 * starts with a clean state next time. 863 */ 864 npxdrop(); 865 critical_exit(); 866 return (_MC_FPOWNED_FPU); 867 } else { 868 critical_exit(); 869 return (_MC_FPOWNED_PCB); 870 } 871} 872 873void 874npxuserinited(struct thread *td) 875{ 876 struct pcb *pcb; 877 878 pcb = td->td_pcb; 879 if (PCB_USER_FPU(pcb)) 880 pcb->pcb_flags |= PCB_NPXINITDONE; 881 pcb->pcb_flags |= PCB_NPXUSERINITDONE; 882} 883 884 885void 886npxsetregs(struct thread *td, union savefpu *addr) 887{ 888 struct pcb *pcb; 889 890 if (!hw_float) 891 return; 892 893 pcb = td->td_pcb; 894 critical_enter(); 895 if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { 896#ifdef CPU_ENABLE_SSE 897 if (!cpu_fxsr) 898#endif 899 fnclex(); /* As in npxdrop(). */ 900 if (((uintptr_t)addr & 0xf) != 0) { 901 bcopy(addr, &pcb->pcb_user_save, sizeof(*addr)); 902 fpurstor(&pcb->pcb_user_save); 903 } else 904 fpurstor(addr); 905 critical_exit(); 906 pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE; 907 } else { 908 critical_exit(); 909 bcopy(addr, &pcb->pcb_user_save, sizeof(*addr)); 910 npxuserinited(td); 911 } 912} 913 914static void 915fpusave(addr) 916 union savefpu *addr; 917{ 918 919#ifdef CPU_ENABLE_SSE 920 if (cpu_fxsr) 921 fxsave(addr); 922 else 923#endif 924 fnsave(addr); 925} 926 927#ifdef CPU_ENABLE_SSE 928/* 929 * On AuthenticAMD processors, the fxrstor instruction does not restore 930 * the x87's stored last instruction pointer, last data pointer, and last 931 * opcode values, except in the rare case in which the exception summary 932 * (ES) bit in the x87 status word is set to 1. 933 * 934 * In order to avoid leaking this information across processes, we clean 935 * these values by performing a dummy load before executing fxrstor(). 936 */ 937static void 938fpu_clean_state(void) 939{ 940 static float dummy_variable = 0.0; 941 u_short status; 942 943 /* 944 * Clear the ES bit in the x87 status word if it is currently 945 * set, in order to avoid causing a fault in the upcoming load. 946 */ 947 fnstsw(&status); 948 if (status & 0x80) 949 fnclex(); 950 951 /* 952 * Load the dummy variable into the x87 stack. This mangles 953 * the x87 stack, but we don't care since we're about to call 954 * fxrstor() anyway. 955 */ 956 __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable)); 957} 958#endif /* CPU_ENABLE_SSE */ 959 960static void 961fpurstor(addr) 962 union savefpu *addr; 963{ 964 965#ifdef CPU_ENABLE_SSE 966 if (cpu_fxsr) 967 fxrstor(addr); 968 else 969#endif 970 frstor(addr); 971} 972 973static device_method_t npx_methods[] = { 974 /* Device interface */ 975 DEVMETHOD(device_identify, npx_identify), 976 DEVMETHOD(device_probe, npx_probe), 977 DEVMETHOD(device_attach, npx_attach), 978 DEVMETHOD(device_detach, bus_generic_detach), 979 DEVMETHOD(device_shutdown, bus_generic_shutdown), 980 DEVMETHOD(device_suspend, bus_generic_suspend), 981 DEVMETHOD(device_resume, bus_generic_resume), 982 983 { 0, 0 } 984}; 985 986static driver_t npx_driver = { 987 "npx", 988 npx_methods, 989 1, /* no softc */ 990}; 991 992static devclass_t npx_devclass; 993 994/* 995 * We prefer to attach to the root nexus so that the usual case (exception 16) 996 * doesn't describe the processor as being `on isa'. 997 */ 998DRIVER_MODULE(npx, nexus, npx_driver, npx_devclass, 0, 0); 999 1000#ifdef DEV_ISA 1001/* 1002 * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI. 1003 */ 1004static struct isa_pnp_id npxisa_ids[] = { 1005 { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */ 1006 { 0 } 1007}; 1008 1009static int 1010npxisa_probe(device_t dev) 1011{ 1012 int result; 1013 if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) { 1014 device_quiet(dev); 1015 } 1016 return(result); 1017} 1018 1019static int 1020npxisa_attach(device_t dev) 1021{ 1022 return (0); 1023} 1024 1025static device_method_t npxisa_methods[] = { 1026 /* Device interface */ 1027 DEVMETHOD(device_probe, npxisa_probe), 1028 DEVMETHOD(device_attach, npxisa_attach), 1029 DEVMETHOD(device_detach, bus_generic_detach), 1030 DEVMETHOD(device_shutdown, bus_generic_shutdown), 1031 DEVMETHOD(device_suspend, bus_generic_suspend), 1032 DEVMETHOD(device_resume, bus_generic_resume), 1033 1034 { 0, 0 } 1035}; 1036 1037static driver_t npxisa_driver = { 1038 "npxisa", 1039 npxisa_methods, 1040 1, /* no softc */ 1041}; 1042 1043static devclass_t npxisa_devclass; 1044 1045DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0); 1046#ifndef PC98 1047DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0); 1048#endif 1049#endif /* DEV_ISA */ 1050 1051static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", 1052 "Kernel contexts for FPU state"); 1053 1054#define XSAVE_AREA_ALIGN 64 1055 1056#define FPU_KERN_CTX_NPXINITDONE 0x01 1057#define FPU_KERN_CTX_DUMMY 0x02 1058 1059struct fpu_kern_ctx { 1060 union savefpu *prev; 1061 uint32_t flags; 1062 char hwstate1[]; 1063}; 1064 1065struct fpu_kern_ctx * 1066fpu_kern_alloc_ctx(u_int flags) 1067{ 1068 struct fpu_kern_ctx *res; 1069 size_t sz; 1070 1071 sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN + 1072 sizeof(union savefpu); 1073 res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? 1074 M_NOWAIT : M_WAITOK) | M_ZERO); 1075 return (res); 1076} 1077 1078void 1079fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) 1080{ 1081 1082 /* XXXKIB clear the memory ? */ 1083 free(ctx, M_FPUKERN_CTX); 1084} 1085 1086static union savefpu * 1087fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx) 1088{ 1089 vm_offset_t p; 1090 1091 p = (vm_offset_t)&ctx->hwstate1; 1092 p = roundup2(p, XSAVE_AREA_ALIGN); 1093 return ((union savefpu *)p); 1094} 1095 1096int 1097fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) 1098{ 1099 struct pcb *pcb; 1100 1101 if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { 1102 ctx->flags = FPU_KERN_CTX_DUMMY; 1103 return (0); 1104 } 1105 pcb = td->td_pcb; 1106 KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == &pcb->pcb_user_save, 1107 ("mangled pcb_save")); 1108 ctx->flags = 0; 1109 if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0) 1110 ctx->flags |= FPU_KERN_CTX_NPXINITDONE; 1111 npxexit(td); 1112 ctx->prev = pcb->pcb_save; 1113 pcb->pcb_save = fpu_kern_ctx_savefpu(ctx); 1114 pcb->pcb_flags |= PCB_KERNNPX; 1115 pcb->pcb_flags &= ~PCB_NPXINITDONE; 1116 return (0); 1117} 1118 1119int 1120fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) 1121{ 1122 struct pcb *pcb; 1123 1124 if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) 1125 return (0); 1126 pcb = td->td_pcb; 1127 critical_enter(); 1128 if (curthread == PCPU_GET(fpcurthread)) 1129 npxdrop(); 1130 critical_exit(); 1131 pcb->pcb_save = ctx->prev; 1132 if (pcb->pcb_save == &pcb->pcb_user_save) { 1133 if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0) 1134 pcb->pcb_flags |= PCB_NPXINITDONE; 1135 else 1136 pcb->pcb_flags &= ~PCB_NPXINITDONE; 1137 pcb->pcb_flags &= ~PCB_KERNNPX; 1138 } else { 1139 if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0) 1140 pcb->pcb_flags |= PCB_NPXINITDONE; 1141 else 1142 pcb->pcb_flags &= ~PCB_NPXINITDONE; 1143 KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); 1144 } 1145 return (0); 1146} 1147 1148int 1149fpu_kern_thread(u_int flags) 1150{ 1151 struct pcb *pcb; 1152 1153 pcb = curpcb; 1154 KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, 1155 ("Only kthread may use fpu_kern_thread")); 1156 KASSERT(curpcb->pcb_save == &curpcb->pcb_user_save, 1157 ("mangled pcb_save")); 1158 KASSERT(PCB_USER_FPU(curpcb), ("recursive call")); 1159 1160 curpcb->pcb_flags |= PCB_KERNNPX; 1161 return (0); 1162} 1163 1164int 1165is_fpu_kern_thread(u_int flags) 1166{ 1167 1168 if ((curthread->td_pflags & TDP_KTHREAD) == 0) 1169 return (0); 1170 return ((curpcb->pcb_flags & PCB_KERNNPX) != 0); 1171} 1172