35
36#include "opt_cpu.h"
37#include "opt_isa.h"
38#include "opt_npx.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/kernel.h>
44#include <sys/lock.h>
45#include <sys/malloc.h>
46#include <sys/module.h>
47#include <sys/mutex.h>
48#include <sys/mutex.h>
49#include <sys/proc.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <machine/bus.h>
53#include <sys/rman.h>
54#ifdef NPX_DEBUG
55#include <sys/syslog.h>
56#endif
57#include <sys/signalvar.h>
58
59#include <machine/asmacros.h>
60#include <machine/cputypes.h>
61#include <machine/frame.h>
62#include <machine/md_var.h>
63#include <machine/pcb.h>
64#include <machine/psl.h>
65#include <machine/resource.h>
66#include <machine/specialreg.h>
67#include <machine/segments.h>
68#include <machine/ucontext.h>
69
70#include <machine/intr_machdep.h>
71#ifdef DEV_ISA
72#include <isa/isavar.h>
73#endif
74
75#if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
76#define CPU_ENABLE_SSE
77#endif
78
79/*
80 * 387 and 287 Numeric Coprocessor Extension (NPX) Driver.
81 */
82
83/* Configuration flags. */
84#define NPX_DISABLE_I586_OPTIMIZED_BCOPY (1 << 0)
85#define NPX_DISABLE_I586_OPTIMIZED_BZERO (1 << 1)
86#define NPX_DISABLE_I586_OPTIMIZED_COPYIO (1 << 2)
87
88#if defined(__GNUCLIKE_ASM) && !defined(lint)
89
90#define fldcw(addr) __asm("fldcw %0" : : "m" (*(addr)))
91#define fnclex() __asm("fnclex")
92#define fninit() __asm("fninit")
93#define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr)))
94#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr)))
95#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=m" (*(addr)))
96#define fp_divide_by_0() __asm("fldz; fld1; fdiv %st,%st(1); fnop")
97#define frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
98#ifdef CPU_ENABLE_SSE
99#define fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
100#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr)))
101#define ldmxcsr(__csr) __asm __volatile("ldmxcsr %0" : : "m" (__csr))
102#endif
103#define start_emulating() __asm("smsw %%ax; orb %0,%%al; lmsw %%ax" \
104 : : "n" (CR0_TS) : "ax")
105#define stop_emulating() __asm("clts")
106
107#else /* !(__GNUCLIKE_ASM && !lint) */
108
109void fldcw(caddr_t addr);
110void fnclex(void);
111void fninit(void);
112void fnsave(caddr_t addr);
113void fnstcw(caddr_t addr);
114void fnstsw(caddr_t addr);
115void fp_divide_by_0(void);
116void frstor(caddr_t addr);
117#ifdef CPU_ENABLE_SSE
118void fxsave(caddr_t addr);
119void fxrstor(caddr_t addr);
120#endif
121void start_emulating(void);
122void stop_emulating(void);
123
124#endif /* __GNUCLIKE_ASM && !lint */
125
126#ifdef CPU_ENABLE_SSE
127#define GET_FPU_CW(thread) \
128 (cpu_fxsr ? \
129 (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_cw : \
130 (thread)->td_pcb->pcb_save.sv_87.sv_env.en_cw)
131#define GET_FPU_SW(thread) \
132 (cpu_fxsr ? \
133 (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_sw : \
134 (thread)->td_pcb->pcb_save.sv_87.sv_env.en_sw)
135#else /* CPU_ENABLE_SSE */
136#define GET_FPU_CW(thread) \
137 (thread->td_pcb->pcb_save.sv_87.sv_env.en_cw)
138#define GET_FPU_SW(thread) \
139 (thread->td_pcb->pcb_save.sv_87.sv_env.en_sw)
140#endif /* CPU_ENABLE_SSE */
141
142typedef u_char bool_t;
143
144#ifdef CPU_ENABLE_SSE
145static void fpu_clean_state(void);
146#endif
147
148static void fpusave(union savefpu *);
149static void fpurstor(union savefpu *);
150static int npx_attach(device_t dev);
151static void npx_identify(driver_t *driver, device_t parent);
152static void npx_intr(void *);
153static int npx_probe(device_t dev);
154#ifdef I586_CPU_XXX
155static long timezero(const char *funcname,
156 void (*func)(void *buf, size_t len));
157#endif /* I586_CPU */
158
159int hw_float; /* XXX currently just alias for npx_exists */
160
161SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint,
162 CTLFLAG_RD, &hw_float, 0,
163 "Floatingpoint instructions executed in hardware");
164
165static volatile u_int npx_intrs_while_probing;
166static volatile u_int npx_traps_while_probing;
167
168static union savefpu npx_cleanstate;
169static bool_t npx_cleanstate_ready;
170static bool_t npx_ex16;
171static bool_t npx_exists;
172static bool_t npx_irq13;
173
174alias_for_inthand_t probetrap;
175__asm(" \n\
176 .text \n\
177 .p2align 2,0x90 \n\
178 .type " __XSTRING(CNAME(probetrap)) ",@function \n\
179" __XSTRING(CNAME(probetrap)) ": \n\
180 ss \n\
181 incl " __XSTRING(CNAME(npx_traps_while_probing)) " \n\
182 fnclex \n\
183 iret \n\
184");
185
186/*
187 * Identify routine. Create a connection point on our parent for probing.
188 */
189static void
190npx_identify(driver, parent)
191 driver_t *driver;
192 device_t parent;
193{
194 device_t child;
195
196 child = BUS_ADD_CHILD(parent, 0, "npx", 0);
197 if (child == NULL)
198 panic("npx_identify");
199}
200
201/*
202 * Do minimal handling of npx interrupts to convert them to traps.
203 */
204static void
205npx_intr(dummy)
206 void *dummy;
207{
208 struct thread *td;
209
210 npx_intrs_while_probing++;
211
212 /*
213 * The BUSY# latch must be cleared in all cases so that the next
214 * unmasked npx exception causes an interrupt.
215 */
216 outb(IO_NPX, 0);
217
218 /*
219 * fpcurthread is normally non-null here. In that case, schedule an
220 * AST to finish the exception handling in the correct context
221 * (this interrupt may occur after the thread has entered the
222 * kernel via a syscall or an interrupt). Otherwise, the npx
223 * state of the thread that caused this interrupt must have been
224 * pushed to the thread's pcb, and clearing of the busy latch
225 * above has finished the (essentially null) handling of this
226 * interrupt. Control will eventually return to the instruction
227 * that caused it and it will repeat. We will eventually (usually
228 * soon) win the race to handle the interrupt properly.
229 */
230 td = PCPU_GET(fpcurthread);
231 if (td != NULL) {
232 td->td_pcb->pcb_flags |= PCB_NPXTRAP;
233 mtx_lock_spin(&sched_lock);
234 td->td_flags |= TDF_ASTPENDING;
235 mtx_unlock_spin(&sched_lock);
236 }
237}
238
239/*
240 * Probe routine. Set flags to tell npxattach() what to do. Set up an
241 * interrupt handler if npx needs to use interrupts.
242 */
243static int
244npx_probe(dev)
245 device_t dev;
246{
247 struct gate_descriptor save_idt_npxtrap;
248 struct resource *ioport_res, *irq_res;
249 void *irq_cookie;
250 int ioport_rid, irq_num, irq_rid;
251 u_short control;
252 u_short status;
253
254 device_set_desc(dev, "math processor");
255
256 /*
257 * Modern CPUs all have an FPU that uses the INT16 interface
258 * and provide a simple way to verify that, so handle the
259 * common case right away.
260 */
261 if (cpu_feature & CPUID_FPU) {
262 hw_float = npx_exists = 1;
263 npx_ex16 = 1;
264 device_quiet(dev);
265 return (0);
266 }
267
268 save_idt_npxtrap = idt[IDT_MF];
269 setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL,
270 GSEL(GCODE_SEL, SEL_KPL));
271 ioport_rid = 0;
272 ioport_res = bus_alloc_resource(dev, SYS_RES_IOPORT, &ioport_rid,
273 IO_NPX, IO_NPX + IO_NPXSIZE - 1, IO_NPXSIZE, RF_ACTIVE);
274 if (ioport_res == NULL)
275 panic("npx: can't get ports");
276 if (resource_int_value("npx", 0, "irq", &irq_num) != 0)
277 irq_num = IRQ_NPX;
278 irq_rid = 0;
279 irq_res = bus_alloc_resource(dev, SYS_RES_IRQ, &irq_rid, irq_num,
280 irq_num, 1, RF_ACTIVE);
281 if (irq_res != NULL) {
282 if (bus_setup_intr(dev, irq_res, INTR_TYPE_MISC | INTR_FAST,
283 npx_intr, NULL, &irq_cookie) != 0)
284 panic("npx: can't create intr");
285 }
286
287 /*
288 * Partially reset the coprocessor, if any. Some BIOS's don't reset
289 * it after a warm boot.
290 */
291 npx_full_reset();
292 outb(IO_NPX, 0);
293
294 /*
295 * Don't trap while we're probing.
296 */
297 stop_emulating();
298
299 /*
300 * Finish resetting the coprocessor, if any. If there is an error
301 * pending, then we may get a bogus IRQ13, but npx_intr() will handle
302 * it OK. Bogus halts have never been observed, but we enabled
303 * IRQ13 and cleared the BUSY# latch early to handle them anyway.
304 */
305 fninit();
306
307 /*
308 * Don't use fwait here because it might hang.
309 * Don't use fnop here because it usually hangs if there is no FPU.
310 */
311 DELAY(1000); /* wait for any IRQ13 */
312#ifdef DIAGNOSTIC
313 if (npx_intrs_while_probing != 0)
314 printf("fninit caused %u bogus npx interrupt(s)\n",
315 npx_intrs_while_probing);
316 if (npx_traps_while_probing != 0)
317 printf("fninit caused %u bogus npx trap(s)\n",
318 npx_traps_while_probing);
319#endif
320 /*
321 * Check for a status of mostly zero.
322 */
323 status = 0x5a5a;
324 fnstsw(&status);
325 if ((status & 0xb8ff) == 0) {
326 /*
327 * Good, now check for a proper control word.
328 */
329 control = 0x5a5a;
330 fnstcw(&control);
331 if ((control & 0x1f3f) == 0x033f) {
332 hw_float = npx_exists = 1;
333 /*
334 * We have an npx, now divide by 0 to see if exception
335 * 16 works.
336 */
337 control &= ~(1 << 2); /* enable divide by 0 trap */
338 fldcw(&control);
339#ifdef FPU_ERROR_BROKEN
340 /*
341 * FPU error signal doesn't work on some CPU
342 * accelerator board.
343 */
344 npx_ex16 = 1;
345 return (0);
346#endif
347 npx_traps_while_probing = npx_intrs_while_probing = 0;
348 fp_divide_by_0();
349 DELAY(1000); /* wait for any IRQ13 */
350 if (npx_traps_while_probing != 0) {
351 /*
352 * Good, exception 16 works.
353 */
354 npx_ex16 = 1;
355 goto no_irq13;
356 }
357 if (npx_intrs_while_probing != 0) {
358 /*
359 * Bad, we are stuck with IRQ13.
360 */
361 npx_irq13 = 1;
362 idt[IDT_MF] = save_idt_npxtrap;
363#ifdef SMP
364 if (mp_ncpus > 1)
365 panic("npx0 cannot use IRQ 13 on an SMP system");
366#endif
367 return (0);
368 }
369 /*
370 * Worse, even IRQ13 is broken. Use emulator.
371 */
372 }
373 }
374 /*
375 * Probe failed, but we want to get to npxattach to initialize the
376 * emulator and say that it has been installed. XXX handle devices
377 * that aren't really devices better.
378 */
379#ifdef SMP
380 if (mp_ncpus > 1)
381 panic("npx0 cannot be emulated on an SMP system");
382#endif
383 /* FALLTHROUGH */
384no_irq13:
385 idt[IDT_MF] = save_idt_npxtrap;
386 if (irq_res != NULL) {
387 bus_teardown_intr(dev, irq_res, irq_cookie);
388 bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
389 }
390 bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res);
391 return (0);
392}
393
394/*
395 * Attach routine - announce which it is, and wire into system
396 */
397static int
398npx_attach(dev)
399 device_t dev;
400{
401 int flags;
402 register_t s;
403
404 flags = device_get_flags(dev);
405
406 if (npx_irq13)
407 device_printf(dev, "IRQ 13 interface\n");
408 else if (!npx_ex16)
409 device_printf(dev, "WARNING: no FPU!\n");
410 else if (!device_is_quiet(dev) || bootverbose)
411 device_printf(dev, "INT 16 interface\n");
412
413 npxinit(__INITIAL_NPXCW__);
414
415 if (npx_cleanstate_ready == 0) {
416 s = intr_disable();
417 stop_emulating();
418 fpusave(&npx_cleanstate);
419 start_emulating();
420 npx_cleanstate_ready = 1;
421 intr_restore(s);
422 }
423#ifdef I586_CPU_XXX
424 if (cpu_class == CPUCLASS_586 && npx_ex16 && npx_exists &&
425 timezero("i586_bzero()", i586_bzero) <
426 timezero("bzero()", bzero) * 4 / 5) {
427 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY))
428 bcopy_vector = i586_bcopy;
429 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BZERO))
430 bzero_vector = i586_bzero;
431 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_COPYIO)) {
432 copyin_vector = i586_copyin;
433 copyout_vector = i586_copyout;
434 }
435 }
436#endif
437
438 return (0); /* XXX unused */
439}
440
441/*
442 * Initialize floating point unit.
443 */
444void
445npxinit(control)
446 u_short control;
447{
448 static union savefpu dummy;
449 register_t savecrit;
450
451 if (!npx_exists)
452 return;
453 /*
454 * fninit has the same h/w bugs as fnsave. Use the detoxified
455 * fnsave to throw away any junk in the fpu. npxsave() initializes
456 * the fpu and sets fpcurthread = NULL as important side effects.
457 */
458 savecrit = intr_disable();
459 npxsave(&dummy);
460 stop_emulating();
461#ifdef CPU_ENABLE_SSE
462 /* XXX npxsave() doesn't actually initialize the fpu in the SSE case. */
463 if (cpu_fxsr)
464 fninit();
465#endif
466 fldcw(&control);
467 start_emulating();
468 intr_restore(savecrit);
469}
470
471/*
472 * Free coprocessor (if we have it).
473 */
474void
475npxexit(td)
476 struct thread *td;
477{
478 register_t savecrit;
479
480 savecrit = intr_disable();
481 if (curthread == PCPU_GET(fpcurthread))
482 npxsave(&PCPU_GET(curpcb)->pcb_save);
483 intr_restore(savecrit);
484#ifdef NPX_DEBUG
485 if (npx_exists) {
486 u_int masked_exceptions;
487
488 masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f;
489 /*
490 * Log exceptions that would have trapped with the old
491 * control word (overflow, divide by 0, and invalid operand).
492 */
493 if (masked_exceptions & 0x0d)
494 log(LOG_ERR,
495 "pid %d (%s) exited with masked floating point exceptions 0x%02x\n",
496 td->td_proc->p_pid, td->td_proc->p_comm,
497 masked_exceptions);
498 }
499#endif
500}
501
502int
503npxformat()
504{
505
506 if (!npx_exists)
507 return (_MC_FPFMT_NODEV);
508#ifdef CPU_ENABLE_SSE
509 if (cpu_fxsr)
510 return (_MC_FPFMT_XMM);
511#endif
512 return (_MC_FPFMT_387);
513}
514
515/*
516 * The following mechanism is used to ensure that the FPE_... value
517 * that is passed as a trapcode to the signal handler of the user
518 * process does not have more than one bit set.
519 *
520 * Multiple bits may be set if the user process modifies the control
521 * word while a status word bit is already set. While this is a sign
522 * of bad coding, we have no choise than to narrow them down to one
523 * bit, since we must not send a trapcode that is not exactly one of
524 * the FPE_ macros.
525 *
526 * The mechanism has a static table with 127 entries. Each combination
527 * of the 7 FPU status word exception bits directly translates to a
528 * position in this table, where a single FPE_... value is stored.
529 * This FPE_... value stored there is considered the "most important"
530 * of the exception bits and will be sent as the signal code. The
531 * precedence of the bits is based upon Intel Document "Numerical
532 * Applications", Chapter "Special Computational Situations".
533 *
534 * The macro to choose one of these values does these steps: 1) Throw
535 * away status word bits that cannot be masked. 2) Throw away the bits
536 * currently masked in the control word, assuming the user isn't
537 * interested in them anymore. 3) Reinsert status word bit 7 (stack
538 * fault) if it is set, which cannot be masked but must be presered.
539 * 4) Use the remaining bits to point into the trapcode table.
540 *
541 * The 6 maskable bits in order of their preference, as stated in the
542 * above referenced Intel manual:
543 * 1 Invalid operation (FP_X_INV)
544 * 1a Stack underflow
545 * 1b Stack overflow
546 * 1c Operand of unsupported format
547 * 1d SNaN operand.
548 * 2 QNaN operand (not an exception, irrelavant here)
549 * 3 Any other invalid-operation not mentioned above or zero divide
550 * (FP_X_INV, FP_X_DZ)
551 * 4 Denormal operand (FP_X_DNML)
552 * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL)
553 * 6 Inexact result (FP_X_IMP)
554 */
555static char fpetable[128] = {
556 0,
557 FPE_FLTINV, /* 1 - INV */
558 FPE_FLTUND, /* 2 - DNML */
559 FPE_FLTINV, /* 3 - INV | DNML */
560 FPE_FLTDIV, /* 4 - DZ */
561 FPE_FLTINV, /* 5 - INV | DZ */
562 FPE_FLTDIV, /* 6 - DNML | DZ */
563 FPE_FLTINV, /* 7 - INV | DNML | DZ */
564 FPE_FLTOVF, /* 8 - OFL */
565 FPE_FLTINV, /* 9 - INV | OFL */
566 FPE_FLTUND, /* A - DNML | OFL */
567 FPE_FLTINV, /* B - INV | DNML | OFL */
568 FPE_FLTDIV, /* C - DZ | OFL */
569 FPE_FLTINV, /* D - INV | DZ | OFL */
570 FPE_FLTDIV, /* E - DNML | DZ | OFL */
571 FPE_FLTINV, /* F - INV | DNML | DZ | OFL */
572 FPE_FLTUND, /* 10 - UFL */
573 FPE_FLTINV, /* 11 - INV | UFL */
574 FPE_FLTUND, /* 12 - DNML | UFL */
575 FPE_FLTINV, /* 13 - INV | DNML | UFL */
576 FPE_FLTDIV, /* 14 - DZ | UFL */
577 FPE_FLTINV, /* 15 - INV | DZ | UFL */
578 FPE_FLTDIV, /* 16 - DNML | DZ | UFL */
579 FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */
580 FPE_FLTOVF, /* 18 - OFL | UFL */
581 FPE_FLTINV, /* 19 - INV | OFL | UFL */
582 FPE_FLTUND, /* 1A - DNML | OFL | UFL */
583 FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */
584 FPE_FLTDIV, /* 1C - DZ | OFL | UFL */
585 FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */
586 FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */
587 FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */
588 FPE_FLTRES, /* 20 - IMP */
589 FPE_FLTINV, /* 21 - INV | IMP */
590 FPE_FLTUND, /* 22 - DNML | IMP */
591 FPE_FLTINV, /* 23 - INV | DNML | IMP */
592 FPE_FLTDIV, /* 24 - DZ | IMP */
593 FPE_FLTINV, /* 25 - INV | DZ | IMP */
594 FPE_FLTDIV, /* 26 - DNML | DZ | IMP */
595 FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */
596 FPE_FLTOVF, /* 28 - OFL | IMP */
597 FPE_FLTINV, /* 29 - INV | OFL | IMP */
598 FPE_FLTUND, /* 2A - DNML | OFL | IMP */
599 FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */
600 FPE_FLTDIV, /* 2C - DZ | OFL | IMP */
601 FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */
602 FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */
603 FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */
604 FPE_FLTUND, /* 30 - UFL | IMP */
605 FPE_FLTINV, /* 31 - INV | UFL | IMP */
606 FPE_FLTUND, /* 32 - DNML | UFL | IMP */
607 FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */
608 FPE_FLTDIV, /* 34 - DZ | UFL | IMP */
609 FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */
610 FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */
611 FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */
612 FPE_FLTOVF, /* 38 - OFL | UFL | IMP */
613 FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */
614 FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */
615 FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */
616 FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */
617 FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */
618 FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */
619 FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */
620 FPE_FLTSUB, /* 40 - STK */
621 FPE_FLTSUB, /* 41 - INV | STK */
622 FPE_FLTUND, /* 42 - DNML | STK */
623 FPE_FLTSUB, /* 43 - INV | DNML | STK */
624 FPE_FLTDIV, /* 44 - DZ | STK */
625 FPE_FLTSUB, /* 45 - INV | DZ | STK */
626 FPE_FLTDIV, /* 46 - DNML | DZ | STK */
627 FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */
628 FPE_FLTOVF, /* 48 - OFL | STK */
629 FPE_FLTSUB, /* 49 - INV | OFL | STK */
630 FPE_FLTUND, /* 4A - DNML | OFL | STK */
631 FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */
632 FPE_FLTDIV, /* 4C - DZ | OFL | STK */
633 FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */
634 FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */
635 FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */
636 FPE_FLTUND, /* 50 - UFL | STK */
637 FPE_FLTSUB, /* 51 - INV | UFL | STK */
638 FPE_FLTUND, /* 52 - DNML | UFL | STK */
639 FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */
640 FPE_FLTDIV, /* 54 - DZ | UFL | STK */
641 FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */
642 FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */
643 FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */
644 FPE_FLTOVF, /* 58 - OFL | UFL | STK */
645 FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */
646 FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */
647 FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */
648 FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */
649 FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */
650 FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */
651 FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */
652 FPE_FLTRES, /* 60 - IMP | STK */
653 FPE_FLTSUB, /* 61 - INV | IMP | STK */
654 FPE_FLTUND, /* 62 - DNML | IMP | STK */
655 FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */
656 FPE_FLTDIV, /* 64 - DZ | IMP | STK */
657 FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */
658 FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */
659 FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */
660 FPE_FLTOVF, /* 68 - OFL | IMP | STK */
661 FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */
662 FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */
663 FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */
664 FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */
665 FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */
666 FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */
667 FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */
668 FPE_FLTUND, /* 70 - UFL | IMP | STK */
669 FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */
670 FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */
671 FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */
672 FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */
673 FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */
674 FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */
675 FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */
676 FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */
677 FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */
678 FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */
679 FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */
680 FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */
681 FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */
682 FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */
683 FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */
684};
685
686/*
687 * Preserve the FP status word, clear FP exceptions, then generate a SIGFPE.
688 *
689 * Clearing exceptions is necessary mainly to avoid IRQ13 bugs. We now
690 * depend on longjmp() restoring a usable state. Restoring the state
691 * or examining it might fail if we didn't clear exceptions.
692 *
693 * The error code chosen will be one of the FPE_... macros. It will be
694 * sent as the second argument to old BSD-style signal handlers and as
695 * "siginfo_t->si_code" (second argument) to SA_SIGINFO signal handlers.
696 *
697 * XXX the FP state is not preserved across signal handlers. So signal
698 * handlers cannot afford to do FP unless they preserve the state or
699 * longjmp() out. Both preserving the state and longjmp()ing may be
700 * destroyed by IRQ13 bugs. Clearing FP exceptions is not an acceptable
701 * solution for signals other than SIGFPE.
702 */
703int
704npxtrap()
705{
706 register_t savecrit;
707 u_short control, status;
708
709 if (!npx_exists) {
710 printf("npxtrap: fpcurthread = %p, curthread = %p, npx_exists = %d\n",
711 PCPU_GET(fpcurthread), curthread, npx_exists);
712 panic("npxtrap from nowhere");
713 }
714 savecrit = intr_disable();
715
716 /*
717 * Interrupt handling (for another interrupt) may have pushed the
718 * state to memory. Fetch the relevant parts of the state from
719 * wherever they are.
720 */
721 if (PCPU_GET(fpcurthread) != curthread) {
722 control = GET_FPU_CW(curthread);
723 status = GET_FPU_SW(curthread);
724 } else {
725 fnstcw(&control);
726 fnstsw(&status);
727 }
728
729 if (PCPU_GET(fpcurthread) == curthread)
730 fnclex();
731 intr_restore(savecrit);
732 return (fpetable[status & ((~control & 0x3f) | 0x40)]);
733}
734
735/*
736 * Implement device not available (DNA) exception
737 *
738 * It would be better to switch FP context here (if curthread != fpcurthread)
739 * and not necessarily for every context switch, but it is too hard to
740 * access foreign pcb's.
741 */
742
743static int err_count = 0;
744
745int
746npxdna()
747{
748 struct pcb *pcb;
749 register_t s;
750#ifdef CPU_ENABLE_SSE
751 int mxcsr;
752#endif
753 u_short control;
754
755 if (!npx_exists)
756 return (0);
757 if (PCPU_GET(fpcurthread) == curthread) {
758 printf("npxdna: fpcurthread == curthread %d times\n",
759 ++err_count);
760 stop_emulating();
761 return (1);
762 }
763 if (PCPU_GET(fpcurthread) != NULL) {
764 printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n",
765 PCPU_GET(fpcurthread),
766 PCPU_GET(fpcurthread)->td_proc->p_pid,
767 curthread, curthread->td_proc->p_pid);
768 panic("npxdna");
769 }
770 s = intr_disable();
771 stop_emulating();
772 /*
773 * Record new context early in case frstor causes an IRQ13.
774 */
775 PCPU_SET(fpcurthread, curthread);
776 pcb = PCPU_GET(curpcb);
777
778 if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
779 /*
780 * This is the first time this thread has used the FPU or
781 * the PCB doesn't contain a clean FPU state. Explicitly
782 * initialize the FPU and load the default control word.
783 */
784 fninit();
785 control = __INITIAL_NPXCW__;
786 fldcw(&control);
787#ifdef CPU_ENABLE_SSE
788 if (cpu_fxsr) {
789 mxcsr = __INITIAL_MXCSR__;
790 ldmxcsr(mxcsr);
791 }
792#endif
793 pcb->pcb_flags |= PCB_NPXINITDONE;
794 } else {
795 /*
796 * The following frstor may cause an IRQ13 when the state
797 * being restored has a pending error. The error will
798 * appear to have been triggered by the current (npx) user
799 * instruction even when that instruction is a no-wait
800 * instruction that should not trigger an error (e.g.,
801 * fnclex). On at least one 486 system all of the no-wait
802 * instructions are broken the same as frstor, so our
803 * treatment does not amplify the breakage. On at least
804 * one 386/Cyrix 387 system, fnclex works correctly while
805 * frstor and fnsave are broken, so our treatment breaks
806 * fnclex if it is the first FPU instruction after a context
807 * switch.
808 */
809 fpurstor(&pcb->pcb_save);
810 }
811 intr_restore(s);
812
813 return (1);
814}
815
816/*
817 * Wrapper for fnsave instruction, partly to handle hardware bugs. When npx
818 * exceptions are reported via IRQ13, spurious IRQ13's may be triggered by
819 * no-wait npx instructions. See the Intel application note AP-578 for
820 * details. This doesn't cause any additional complications here. IRQ13's
821 * are inherently asynchronous unless the CPU is frozen to deliver them --
822 * one that started in userland may be delivered many instructions later,
823 * after the process has entered the kernel. It may even be delivered after
824 * the fnsave here completes. A spurious IRQ13 for the fnsave is handled in
825 * the same way as a very-late-arriving non-spurious IRQ13 from user mode:
826 * it is normally ignored at first because we set fpcurthread to NULL; it is
827 * normally retriggered in npxdna() after return to user mode.
828 *
829 * npxsave() must be called with interrupts disabled, so that it clears
830 * fpcurthread atomically with saving the state. We require callers to do the
831 * disabling, since most callers need to disable interrupts anyway to call
832 * npxsave() atomically with checking fpcurthread.
833 *
834 * A previous version of npxsave() went to great lengths to excecute fnsave
835 * with interrupts enabled in case executing it froze the CPU. This case
836 * can't happen, at least for Intel CPU/NPX's. Spurious IRQ13's don't imply
837 * spurious freezes.
838 */
839void
840npxsave(addr)
841 union savefpu *addr;
842{
843
844 stop_emulating();
845 fpusave(addr);
846
847 start_emulating();
848 PCPU_SET(fpcurthread, NULL);
849}
850
851/*
852 * This should be called with interrupts disabled and only when the owning
853 * FPU thread is non-null.
854 */
855void
856npxdrop()
857{
858 struct thread *td;
859
860 /*
861 * Discard pending exceptions in the !cpu_fxsr case so that unmasked
862 * ones don't cause a panic on the next frstor.
863 */
864#ifdef CPU_ENABLE_SSE
865 if (!cpu_fxsr)
866#endif
867 fnclex();
868
869 td = PCPU_GET(fpcurthread);
870 PCPU_SET(fpcurthread, NULL);
871 td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE;
872 start_emulating();
873}
874
875/*
876 * Get the state of the FPU without dropping ownership (if possible).
877 * It returns the FPU ownership status.
878 */
879int
880npxgetregs(td, addr)
881 struct thread *td;
882 union savefpu *addr;
883{
884 register_t s;
885
886 if (!npx_exists)
887 return (_MC_FPOWNED_NONE);
888
889 if ((td->td_pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
890 if (npx_cleanstate_ready)
891 bcopy(&npx_cleanstate, addr, sizeof(npx_cleanstate));
892 else
893 bzero(addr, sizeof(*addr));
894 return (_MC_FPOWNED_NONE);
895 }
896 s = intr_disable();
897 if (td == PCPU_GET(fpcurthread)) {
898 fpusave(addr);
899#ifdef CPU_ENABLE_SSE
900 if (!cpu_fxsr)
901#endif
902 /*
903 * fnsave initializes the FPU and destroys whatever
904 * context it contains. Make sure the FPU owner
905 * starts with a clean state next time.
906 */
907 npxdrop();
908 intr_restore(s);
909 return (_MC_FPOWNED_FPU);
910 } else {
911 intr_restore(s);
912 bcopy(&td->td_pcb->pcb_save, addr, sizeof(*addr));
913 return (_MC_FPOWNED_PCB);
914 }
915}
916
917/*
918 * Set the state of the FPU.
919 */
920void
921npxsetregs(td, addr)
922 struct thread *td;
923 union savefpu *addr;
924{
925 register_t s;
926
927 if (!npx_exists)
928 return;
929
930 s = intr_disable();
931 if (td == PCPU_GET(fpcurthread)) {
932#ifdef CPU_ENABLE_SSE
933 if (!cpu_fxsr)
934#endif
935 fnclex(); /* As in npxdrop(). */
936 fpurstor(addr);
937 intr_restore(s);
938 } else {
939 intr_restore(s);
940 bcopy(addr, &td->td_pcb->pcb_save, sizeof(*addr));
941 }
942 curthread->td_pcb->pcb_flags |= PCB_NPXINITDONE;
943}
944
| 35
36#include "opt_cpu.h"
37#include "opt_isa.h"
38#include "opt_npx.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/kernel.h>
44#include <sys/lock.h>
45#include <sys/malloc.h>
46#include <sys/module.h>
47#include <sys/mutex.h>
48#include <sys/mutex.h>
49#include <sys/proc.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <machine/bus.h>
53#include <sys/rman.h>
54#ifdef NPX_DEBUG
55#include <sys/syslog.h>
56#endif
57#include <sys/signalvar.h>
58
59#include <machine/asmacros.h>
60#include <machine/cputypes.h>
61#include <machine/frame.h>
62#include <machine/md_var.h>
63#include <machine/pcb.h>
64#include <machine/psl.h>
65#include <machine/resource.h>
66#include <machine/specialreg.h>
67#include <machine/segments.h>
68#include <machine/ucontext.h>
69
70#include <machine/intr_machdep.h>
71#ifdef DEV_ISA
72#include <isa/isavar.h>
73#endif
74
75#if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
76#define CPU_ENABLE_SSE
77#endif
78
79/*
80 * 387 and 287 Numeric Coprocessor Extension (NPX) Driver.
81 */
82
83/* Configuration flags. */
84#define NPX_DISABLE_I586_OPTIMIZED_BCOPY (1 << 0)
85#define NPX_DISABLE_I586_OPTIMIZED_BZERO (1 << 1)
86#define NPX_DISABLE_I586_OPTIMIZED_COPYIO (1 << 2)
87
88#if defined(__GNUCLIKE_ASM) && !defined(lint)
89
90#define fldcw(addr) __asm("fldcw %0" : : "m" (*(addr)))
91#define fnclex() __asm("fnclex")
92#define fninit() __asm("fninit")
93#define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr)))
94#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr)))
95#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=m" (*(addr)))
96#define fp_divide_by_0() __asm("fldz; fld1; fdiv %st,%st(1); fnop")
97#define frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
98#ifdef CPU_ENABLE_SSE
99#define fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
100#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr)))
101#define ldmxcsr(__csr) __asm __volatile("ldmxcsr %0" : : "m" (__csr))
102#endif
103#define start_emulating() __asm("smsw %%ax; orb %0,%%al; lmsw %%ax" \
104 : : "n" (CR0_TS) : "ax")
105#define stop_emulating() __asm("clts")
106
107#else /* !(__GNUCLIKE_ASM && !lint) */
108
109void fldcw(caddr_t addr);
110void fnclex(void);
111void fninit(void);
112void fnsave(caddr_t addr);
113void fnstcw(caddr_t addr);
114void fnstsw(caddr_t addr);
115void fp_divide_by_0(void);
116void frstor(caddr_t addr);
117#ifdef CPU_ENABLE_SSE
118void fxsave(caddr_t addr);
119void fxrstor(caddr_t addr);
120#endif
121void start_emulating(void);
122void stop_emulating(void);
123
124#endif /* __GNUCLIKE_ASM && !lint */
125
126#ifdef CPU_ENABLE_SSE
127#define GET_FPU_CW(thread) \
128 (cpu_fxsr ? \
129 (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_cw : \
130 (thread)->td_pcb->pcb_save.sv_87.sv_env.en_cw)
131#define GET_FPU_SW(thread) \
132 (cpu_fxsr ? \
133 (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_sw : \
134 (thread)->td_pcb->pcb_save.sv_87.sv_env.en_sw)
135#else /* CPU_ENABLE_SSE */
136#define GET_FPU_CW(thread) \
137 (thread->td_pcb->pcb_save.sv_87.sv_env.en_cw)
138#define GET_FPU_SW(thread) \
139 (thread->td_pcb->pcb_save.sv_87.sv_env.en_sw)
140#endif /* CPU_ENABLE_SSE */
141
142typedef u_char bool_t;
143
144#ifdef CPU_ENABLE_SSE
145static void fpu_clean_state(void);
146#endif
147
148static void fpusave(union savefpu *);
149static void fpurstor(union savefpu *);
150static int npx_attach(device_t dev);
151static void npx_identify(driver_t *driver, device_t parent);
152static void npx_intr(void *);
153static int npx_probe(device_t dev);
154#ifdef I586_CPU_XXX
155static long timezero(const char *funcname,
156 void (*func)(void *buf, size_t len));
157#endif /* I586_CPU */
158
159int hw_float; /* XXX currently just alias for npx_exists */
160
161SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint,
162 CTLFLAG_RD, &hw_float, 0,
163 "Floatingpoint instructions executed in hardware");
164
165static volatile u_int npx_intrs_while_probing;
166static volatile u_int npx_traps_while_probing;
167
168static union savefpu npx_cleanstate;
169static bool_t npx_cleanstate_ready;
170static bool_t npx_ex16;
171static bool_t npx_exists;
172static bool_t npx_irq13;
173
174alias_for_inthand_t probetrap;
175__asm(" \n\
176 .text \n\
177 .p2align 2,0x90 \n\
178 .type " __XSTRING(CNAME(probetrap)) ",@function \n\
179" __XSTRING(CNAME(probetrap)) ": \n\
180 ss \n\
181 incl " __XSTRING(CNAME(npx_traps_while_probing)) " \n\
182 fnclex \n\
183 iret \n\
184");
185
186/*
187 * Identify routine. Create a connection point on our parent for probing.
188 */
189static void
190npx_identify(driver, parent)
191 driver_t *driver;
192 device_t parent;
193{
194 device_t child;
195
196 child = BUS_ADD_CHILD(parent, 0, "npx", 0);
197 if (child == NULL)
198 panic("npx_identify");
199}
200
201/*
202 * Do minimal handling of npx interrupts to convert them to traps.
203 */
204static void
205npx_intr(dummy)
206 void *dummy;
207{
208 struct thread *td;
209
210 npx_intrs_while_probing++;
211
212 /*
213 * The BUSY# latch must be cleared in all cases so that the next
214 * unmasked npx exception causes an interrupt.
215 */
216 outb(IO_NPX, 0);
217
218 /*
219 * fpcurthread is normally non-null here. In that case, schedule an
220 * AST to finish the exception handling in the correct context
221 * (this interrupt may occur after the thread has entered the
222 * kernel via a syscall or an interrupt). Otherwise, the npx
223 * state of the thread that caused this interrupt must have been
224 * pushed to the thread's pcb, and clearing of the busy latch
225 * above has finished the (essentially null) handling of this
226 * interrupt. Control will eventually return to the instruction
227 * that caused it and it will repeat. We will eventually (usually
228 * soon) win the race to handle the interrupt properly.
229 */
230 td = PCPU_GET(fpcurthread);
231 if (td != NULL) {
232 td->td_pcb->pcb_flags |= PCB_NPXTRAP;
233 mtx_lock_spin(&sched_lock);
234 td->td_flags |= TDF_ASTPENDING;
235 mtx_unlock_spin(&sched_lock);
236 }
237}
238
239/*
240 * Probe routine. Set flags to tell npxattach() what to do. Set up an
241 * interrupt handler if npx needs to use interrupts.
242 */
243static int
244npx_probe(dev)
245 device_t dev;
246{
247 struct gate_descriptor save_idt_npxtrap;
248 struct resource *ioport_res, *irq_res;
249 void *irq_cookie;
250 int ioport_rid, irq_num, irq_rid;
251 u_short control;
252 u_short status;
253
254 device_set_desc(dev, "math processor");
255
256 /*
257 * Modern CPUs all have an FPU that uses the INT16 interface
258 * and provide a simple way to verify that, so handle the
259 * common case right away.
260 */
261 if (cpu_feature & CPUID_FPU) {
262 hw_float = npx_exists = 1;
263 npx_ex16 = 1;
264 device_quiet(dev);
265 return (0);
266 }
267
268 save_idt_npxtrap = idt[IDT_MF];
269 setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL,
270 GSEL(GCODE_SEL, SEL_KPL));
271 ioport_rid = 0;
272 ioport_res = bus_alloc_resource(dev, SYS_RES_IOPORT, &ioport_rid,
273 IO_NPX, IO_NPX + IO_NPXSIZE - 1, IO_NPXSIZE, RF_ACTIVE);
274 if (ioport_res == NULL)
275 panic("npx: can't get ports");
276 if (resource_int_value("npx", 0, "irq", &irq_num) != 0)
277 irq_num = IRQ_NPX;
278 irq_rid = 0;
279 irq_res = bus_alloc_resource(dev, SYS_RES_IRQ, &irq_rid, irq_num,
280 irq_num, 1, RF_ACTIVE);
281 if (irq_res != NULL) {
282 if (bus_setup_intr(dev, irq_res, INTR_TYPE_MISC | INTR_FAST,
283 npx_intr, NULL, &irq_cookie) != 0)
284 panic("npx: can't create intr");
285 }
286
287 /*
288 * Partially reset the coprocessor, if any. Some BIOS's don't reset
289 * it after a warm boot.
290 */
291 npx_full_reset();
292 outb(IO_NPX, 0);
293
294 /*
295 * Don't trap while we're probing.
296 */
297 stop_emulating();
298
299 /*
300 * Finish resetting the coprocessor, if any. If there is an error
301 * pending, then we may get a bogus IRQ13, but npx_intr() will handle
302 * it OK. Bogus halts have never been observed, but we enabled
303 * IRQ13 and cleared the BUSY# latch early to handle them anyway.
304 */
305 fninit();
306
307 /*
308 * Don't use fwait here because it might hang.
309 * Don't use fnop here because it usually hangs if there is no FPU.
310 */
311 DELAY(1000); /* wait for any IRQ13 */
312#ifdef DIAGNOSTIC
313 if (npx_intrs_while_probing != 0)
314 printf("fninit caused %u bogus npx interrupt(s)\n",
315 npx_intrs_while_probing);
316 if (npx_traps_while_probing != 0)
317 printf("fninit caused %u bogus npx trap(s)\n",
318 npx_traps_while_probing);
319#endif
320 /*
321 * Check for a status of mostly zero.
322 */
323 status = 0x5a5a;
324 fnstsw(&status);
325 if ((status & 0xb8ff) == 0) {
326 /*
327 * Good, now check for a proper control word.
328 */
329 control = 0x5a5a;
330 fnstcw(&control);
331 if ((control & 0x1f3f) == 0x033f) {
332 hw_float = npx_exists = 1;
333 /*
334 * We have an npx, now divide by 0 to see if exception
335 * 16 works.
336 */
337 control &= ~(1 << 2); /* enable divide by 0 trap */
338 fldcw(&control);
339#ifdef FPU_ERROR_BROKEN
340 /*
341 * FPU error signal doesn't work on some CPU
342 * accelerator board.
343 */
344 npx_ex16 = 1;
345 return (0);
346#endif
347 npx_traps_while_probing = npx_intrs_while_probing = 0;
348 fp_divide_by_0();
349 DELAY(1000); /* wait for any IRQ13 */
350 if (npx_traps_while_probing != 0) {
351 /*
352 * Good, exception 16 works.
353 */
354 npx_ex16 = 1;
355 goto no_irq13;
356 }
357 if (npx_intrs_while_probing != 0) {
358 /*
359 * Bad, we are stuck with IRQ13.
360 */
361 npx_irq13 = 1;
362 idt[IDT_MF] = save_idt_npxtrap;
363#ifdef SMP
364 if (mp_ncpus > 1)
365 panic("npx0 cannot use IRQ 13 on an SMP system");
366#endif
367 return (0);
368 }
369 /*
370 * Worse, even IRQ13 is broken. Use emulator.
371 */
372 }
373 }
374 /*
375 * Probe failed, but we want to get to npxattach to initialize the
376 * emulator and say that it has been installed. XXX handle devices
377 * that aren't really devices better.
378 */
379#ifdef SMP
380 if (mp_ncpus > 1)
381 panic("npx0 cannot be emulated on an SMP system");
382#endif
383 /* FALLTHROUGH */
384no_irq13:
385 idt[IDT_MF] = save_idt_npxtrap;
386 if (irq_res != NULL) {
387 bus_teardown_intr(dev, irq_res, irq_cookie);
388 bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
389 }
390 bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res);
391 return (0);
392}
393
394/*
395 * Attach routine - announce which it is, and wire into system
396 */
397static int
398npx_attach(dev)
399 device_t dev;
400{
401 int flags;
402 register_t s;
403
404 flags = device_get_flags(dev);
405
406 if (npx_irq13)
407 device_printf(dev, "IRQ 13 interface\n");
408 else if (!npx_ex16)
409 device_printf(dev, "WARNING: no FPU!\n");
410 else if (!device_is_quiet(dev) || bootverbose)
411 device_printf(dev, "INT 16 interface\n");
412
413 npxinit(__INITIAL_NPXCW__);
414
415 if (npx_cleanstate_ready == 0) {
416 s = intr_disable();
417 stop_emulating();
418 fpusave(&npx_cleanstate);
419 start_emulating();
420 npx_cleanstate_ready = 1;
421 intr_restore(s);
422 }
423#ifdef I586_CPU_XXX
424 if (cpu_class == CPUCLASS_586 && npx_ex16 && npx_exists &&
425 timezero("i586_bzero()", i586_bzero) <
426 timezero("bzero()", bzero) * 4 / 5) {
427 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY))
428 bcopy_vector = i586_bcopy;
429 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BZERO))
430 bzero_vector = i586_bzero;
431 if (!(flags & NPX_DISABLE_I586_OPTIMIZED_COPYIO)) {
432 copyin_vector = i586_copyin;
433 copyout_vector = i586_copyout;
434 }
435 }
436#endif
437
438 return (0); /* XXX unused */
439}
440
441/*
442 * Initialize floating point unit.
443 */
444void
445npxinit(control)
446 u_short control;
447{
448 static union savefpu dummy;
449 register_t savecrit;
450
451 if (!npx_exists)
452 return;
453 /*
454 * fninit has the same h/w bugs as fnsave. Use the detoxified
455 * fnsave to throw away any junk in the fpu. npxsave() initializes
456 * the fpu and sets fpcurthread = NULL as important side effects.
457 */
458 savecrit = intr_disable();
459 npxsave(&dummy);
460 stop_emulating();
461#ifdef CPU_ENABLE_SSE
462 /* XXX npxsave() doesn't actually initialize the fpu in the SSE case. */
463 if (cpu_fxsr)
464 fninit();
465#endif
466 fldcw(&control);
467 start_emulating();
468 intr_restore(savecrit);
469}
470
471/*
472 * Free coprocessor (if we have it).
473 */
474void
475npxexit(td)
476 struct thread *td;
477{
478 register_t savecrit;
479
480 savecrit = intr_disable();
481 if (curthread == PCPU_GET(fpcurthread))
482 npxsave(&PCPU_GET(curpcb)->pcb_save);
483 intr_restore(savecrit);
484#ifdef NPX_DEBUG
485 if (npx_exists) {
486 u_int masked_exceptions;
487
488 masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f;
489 /*
490 * Log exceptions that would have trapped with the old
491 * control word (overflow, divide by 0, and invalid operand).
492 */
493 if (masked_exceptions & 0x0d)
494 log(LOG_ERR,
495 "pid %d (%s) exited with masked floating point exceptions 0x%02x\n",
496 td->td_proc->p_pid, td->td_proc->p_comm,
497 masked_exceptions);
498 }
499#endif
500}
501
502int
503npxformat()
504{
505
506 if (!npx_exists)
507 return (_MC_FPFMT_NODEV);
508#ifdef CPU_ENABLE_SSE
509 if (cpu_fxsr)
510 return (_MC_FPFMT_XMM);
511#endif
512 return (_MC_FPFMT_387);
513}
514
515/*
516 * The following mechanism is used to ensure that the FPE_... value
517 * that is passed as a trapcode to the signal handler of the user
518 * process does not have more than one bit set.
519 *
520 * Multiple bits may be set if the user process modifies the control
521 * word while a status word bit is already set. While this is a sign
522 * of bad coding, we have no choise than to narrow them down to one
523 * bit, since we must not send a trapcode that is not exactly one of
524 * the FPE_ macros.
525 *
526 * The mechanism has a static table with 127 entries. Each combination
527 * of the 7 FPU status word exception bits directly translates to a
528 * position in this table, where a single FPE_... value is stored.
529 * This FPE_... value stored there is considered the "most important"
530 * of the exception bits and will be sent as the signal code. The
531 * precedence of the bits is based upon Intel Document "Numerical
532 * Applications", Chapter "Special Computational Situations".
533 *
534 * The macro to choose one of these values does these steps: 1) Throw
535 * away status word bits that cannot be masked. 2) Throw away the bits
536 * currently masked in the control word, assuming the user isn't
537 * interested in them anymore. 3) Reinsert status word bit 7 (stack
538 * fault) if it is set, which cannot be masked but must be presered.
539 * 4) Use the remaining bits to point into the trapcode table.
540 *
541 * The 6 maskable bits in order of their preference, as stated in the
542 * above referenced Intel manual:
543 * 1 Invalid operation (FP_X_INV)
544 * 1a Stack underflow
545 * 1b Stack overflow
546 * 1c Operand of unsupported format
547 * 1d SNaN operand.
548 * 2 QNaN operand (not an exception, irrelavant here)
549 * 3 Any other invalid-operation not mentioned above or zero divide
550 * (FP_X_INV, FP_X_DZ)
551 * 4 Denormal operand (FP_X_DNML)
552 * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL)
553 * 6 Inexact result (FP_X_IMP)
554 */
555static char fpetable[128] = {
556 0,
557 FPE_FLTINV, /* 1 - INV */
558 FPE_FLTUND, /* 2 - DNML */
559 FPE_FLTINV, /* 3 - INV | DNML */
560 FPE_FLTDIV, /* 4 - DZ */
561 FPE_FLTINV, /* 5 - INV | DZ */
562 FPE_FLTDIV, /* 6 - DNML | DZ */
563 FPE_FLTINV, /* 7 - INV | DNML | DZ */
564 FPE_FLTOVF, /* 8 - OFL */
565 FPE_FLTINV, /* 9 - INV | OFL */
566 FPE_FLTUND, /* A - DNML | OFL */
567 FPE_FLTINV, /* B - INV | DNML | OFL */
568 FPE_FLTDIV, /* C - DZ | OFL */
569 FPE_FLTINV, /* D - INV | DZ | OFL */
570 FPE_FLTDIV, /* E - DNML | DZ | OFL */
571 FPE_FLTINV, /* F - INV | DNML | DZ | OFL */
572 FPE_FLTUND, /* 10 - UFL */
573 FPE_FLTINV, /* 11 - INV | UFL */
574 FPE_FLTUND, /* 12 - DNML | UFL */
575 FPE_FLTINV, /* 13 - INV | DNML | UFL */
576 FPE_FLTDIV, /* 14 - DZ | UFL */
577 FPE_FLTINV, /* 15 - INV | DZ | UFL */
578 FPE_FLTDIV, /* 16 - DNML | DZ | UFL */
579 FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */
580 FPE_FLTOVF, /* 18 - OFL | UFL */
581 FPE_FLTINV, /* 19 - INV | OFL | UFL */
582 FPE_FLTUND, /* 1A - DNML | OFL | UFL */
583 FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */
584 FPE_FLTDIV, /* 1C - DZ | OFL | UFL */
585 FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */
586 FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */
587 FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */
588 FPE_FLTRES, /* 20 - IMP */
589 FPE_FLTINV, /* 21 - INV | IMP */
590 FPE_FLTUND, /* 22 - DNML | IMP */
591 FPE_FLTINV, /* 23 - INV | DNML | IMP */
592 FPE_FLTDIV, /* 24 - DZ | IMP */
593 FPE_FLTINV, /* 25 - INV | DZ | IMP */
594 FPE_FLTDIV, /* 26 - DNML | DZ | IMP */
595 FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */
596 FPE_FLTOVF, /* 28 - OFL | IMP */
597 FPE_FLTINV, /* 29 - INV | OFL | IMP */
598 FPE_FLTUND, /* 2A - DNML | OFL | IMP */
599 FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */
600 FPE_FLTDIV, /* 2C - DZ | OFL | IMP */
601 FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */
602 FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */
603 FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */
604 FPE_FLTUND, /* 30 - UFL | IMP */
605 FPE_FLTINV, /* 31 - INV | UFL | IMP */
606 FPE_FLTUND, /* 32 - DNML | UFL | IMP */
607 FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */
608 FPE_FLTDIV, /* 34 - DZ | UFL | IMP */
609 FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */
610 FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */
611 FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */
612 FPE_FLTOVF, /* 38 - OFL | UFL | IMP */
613 FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */
614 FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */
615 FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */
616 FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */
617 FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */
618 FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */
619 FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */
620 FPE_FLTSUB, /* 40 - STK */
621 FPE_FLTSUB, /* 41 - INV | STK */
622 FPE_FLTUND, /* 42 - DNML | STK */
623 FPE_FLTSUB, /* 43 - INV | DNML | STK */
624 FPE_FLTDIV, /* 44 - DZ | STK */
625 FPE_FLTSUB, /* 45 - INV | DZ | STK */
626 FPE_FLTDIV, /* 46 - DNML | DZ | STK */
627 FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */
628 FPE_FLTOVF, /* 48 - OFL | STK */
629 FPE_FLTSUB, /* 49 - INV | OFL | STK */
630 FPE_FLTUND, /* 4A - DNML | OFL | STK */
631 FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */
632 FPE_FLTDIV, /* 4C - DZ | OFL | STK */
633 FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */
634 FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */
635 FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */
636 FPE_FLTUND, /* 50 - UFL | STK */
637 FPE_FLTSUB, /* 51 - INV | UFL | STK */
638 FPE_FLTUND, /* 52 - DNML | UFL | STK */
639 FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */
640 FPE_FLTDIV, /* 54 - DZ | UFL | STK */
641 FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */
642 FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */
643 FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */
644 FPE_FLTOVF, /* 58 - OFL | UFL | STK */
645 FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */
646 FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */
647 FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */
648 FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */
649 FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */
650 FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */
651 FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */
652 FPE_FLTRES, /* 60 - IMP | STK */
653 FPE_FLTSUB, /* 61 - INV | IMP | STK */
654 FPE_FLTUND, /* 62 - DNML | IMP | STK */
655 FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */
656 FPE_FLTDIV, /* 64 - DZ | IMP | STK */
657 FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */
658 FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */
659 FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */
660 FPE_FLTOVF, /* 68 - OFL | IMP | STK */
661 FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */
662 FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */
663 FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */
664 FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */
665 FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */
666 FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */
667 FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */
668 FPE_FLTUND, /* 70 - UFL | IMP | STK */
669 FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */
670 FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */
671 FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */
672 FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */
673 FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */
674 FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */
675 FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */
676 FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */
677 FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */
678 FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */
679 FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */
680 FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */
681 FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */
682 FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */
683 FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */
684};
685
686/*
687 * Preserve the FP status word, clear FP exceptions, then generate a SIGFPE.
688 *
689 * Clearing exceptions is necessary mainly to avoid IRQ13 bugs. We now
690 * depend on longjmp() restoring a usable state. Restoring the state
691 * or examining it might fail if we didn't clear exceptions.
692 *
693 * The error code chosen will be one of the FPE_... macros. It will be
694 * sent as the second argument to old BSD-style signal handlers and as
695 * "siginfo_t->si_code" (second argument) to SA_SIGINFO signal handlers.
696 *
697 * XXX the FP state is not preserved across signal handlers. So signal
698 * handlers cannot afford to do FP unless they preserve the state or
699 * longjmp() out. Both preserving the state and longjmp()ing may be
700 * destroyed by IRQ13 bugs. Clearing FP exceptions is not an acceptable
701 * solution for signals other than SIGFPE.
702 */
703int
704npxtrap()
705{
706 register_t savecrit;
707 u_short control, status;
708
709 if (!npx_exists) {
710 printf("npxtrap: fpcurthread = %p, curthread = %p, npx_exists = %d\n",
711 PCPU_GET(fpcurthread), curthread, npx_exists);
712 panic("npxtrap from nowhere");
713 }
714 savecrit = intr_disable();
715
716 /*
717 * Interrupt handling (for another interrupt) may have pushed the
718 * state to memory. Fetch the relevant parts of the state from
719 * wherever they are.
720 */
721 if (PCPU_GET(fpcurthread) != curthread) {
722 control = GET_FPU_CW(curthread);
723 status = GET_FPU_SW(curthread);
724 } else {
725 fnstcw(&control);
726 fnstsw(&status);
727 }
728
729 if (PCPU_GET(fpcurthread) == curthread)
730 fnclex();
731 intr_restore(savecrit);
732 return (fpetable[status & ((~control & 0x3f) | 0x40)]);
733}
734
735/*
736 * Implement device not available (DNA) exception
737 *
738 * It would be better to switch FP context here (if curthread != fpcurthread)
739 * and not necessarily for every context switch, but it is too hard to
740 * access foreign pcb's.
741 */
742
743static int err_count = 0;
744
745int
746npxdna()
747{
748 struct pcb *pcb;
749 register_t s;
750#ifdef CPU_ENABLE_SSE
751 int mxcsr;
752#endif
753 u_short control;
754
755 if (!npx_exists)
756 return (0);
757 if (PCPU_GET(fpcurthread) == curthread) {
758 printf("npxdna: fpcurthread == curthread %d times\n",
759 ++err_count);
760 stop_emulating();
761 return (1);
762 }
763 if (PCPU_GET(fpcurthread) != NULL) {
764 printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n",
765 PCPU_GET(fpcurthread),
766 PCPU_GET(fpcurthread)->td_proc->p_pid,
767 curthread, curthread->td_proc->p_pid);
768 panic("npxdna");
769 }
770 s = intr_disable();
771 stop_emulating();
772 /*
773 * Record new context early in case frstor causes an IRQ13.
774 */
775 PCPU_SET(fpcurthread, curthread);
776 pcb = PCPU_GET(curpcb);
777
778 if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
779 /*
780 * This is the first time this thread has used the FPU or
781 * the PCB doesn't contain a clean FPU state. Explicitly
782 * initialize the FPU and load the default control word.
783 */
784 fninit();
785 control = __INITIAL_NPXCW__;
786 fldcw(&control);
787#ifdef CPU_ENABLE_SSE
788 if (cpu_fxsr) {
789 mxcsr = __INITIAL_MXCSR__;
790 ldmxcsr(mxcsr);
791 }
792#endif
793 pcb->pcb_flags |= PCB_NPXINITDONE;
794 } else {
795 /*
796 * The following frstor may cause an IRQ13 when the state
797 * being restored has a pending error. The error will
798 * appear to have been triggered by the current (npx) user
799 * instruction even when that instruction is a no-wait
800 * instruction that should not trigger an error (e.g.,
801 * fnclex). On at least one 486 system all of the no-wait
802 * instructions are broken the same as frstor, so our
803 * treatment does not amplify the breakage. On at least
804 * one 386/Cyrix 387 system, fnclex works correctly while
805 * frstor and fnsave are broken, so our treatment breaks
806 * fnclex if it is the first FPU instruction after a context
807 * switch.
808 */
809 fpurstor(&pcb->pcb_save);
810 }
811 intr_restore(s);
812
813 return (1);
814}
815
816/*
817 * Wrapper for fnsave instruction, partly to handle hardware bugs. When npx
818 * exceptions are reported via IRQ13, spurious IRQ13's may be triggered by
819 * no-wait npx instructions. See the Intel application note AP-578 for
820 * details. This doesn't cause any additional complications here. IRQ13's
821 * are inherently asynchronous unless the CPU is frozen to deliver them --
822 * one that started in userland may be delivered many instructions later,
823 * after the process has entered the kernel. It may even be delivered after
824 * the fnsave here completes. A spurious IRQ13 for the fnsave is handled in
825 * the same way as a very-late-arriving non-spurious IRQ13 from user mode:
826 * it is normally ignored at first because we set fpcurthread to NULL; it is
827 * normally retriggered in npxdna() after return to user mode.
828 *
829 * npxsave() must be called with interrupts disabled, so that it clears
830 * fpcurthread atomically with saving the state. We require callers to do the
831 * disabling, since most callers need to disable interrupts anyway to call
832 * npxsave() atomically with checking fpcurthread.
833 *
834 * A previous version of npxsave() went to great lengths to excecute fnsave
835 * with interrupts enabled in case executing it froze the CPU. This case
836 * can't happen, at least for Intel CPU/NPX's. Spurious IRQ13's don't imply
837 * spurious freezes.
838 */
839void
840npxsave(addr)
841 union savefpu *addr;
842{
843
844 stop_emulating();
845 fpusave(addr);
846
847 start_emulating();
848 PCPU_SET(fpcurthread, NULL);
849}
850
851/*
852 * This should be called with interrupts disabled and only when the owning
853 * FPU thread is non-null.
854 */
855void
856npxdrop()
857{
858 struct thread *td;
859
860 /*
861 * Discard pending exceptions in the !cpu_fxsr case so that unmasked
862 * ones don't cause a panic on the next frstor.
863 */
864#ifdef CPU_ENABLE_SSE
865 if (!cpu_fxsr)
866#endif
867 fnclex();
868
869 td = PCPU_GET(fpcurthread);
870 PCPU_SET(fpcurthread, NULL);
871 td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE;
872 start_emulating();
873}
874
875/*
876 * Get the state of the FPU without dropping ownership (if possible).
877 * It returns the FPU ownership status.
878 */
879int
880npxgetregs(td, addr)
881 struct thread *td;
882 union savefpu *addr;
883{
884 register_t s;
885
886 if (!npx_exists)
887 return (_MC_FPOWNED_NONE);
888
889 if ((td->td_pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
890 if (npx_cleanstate_ready)
891 bcopy(&npx_cleanstate, addr, sizeof(npx_cleanstate));
892 else
893 bzero(addr, sizeof(*addr));
894 return (_MC_FPOWNED_NONE);
895 }
896 s = intr_disable();
897 if (td == PCPU_GET(fpcurthread)) {
898 fpusave(addr);
899#ifdef CPU_ENABLE_SSE
900 if (!cpu_fxsr)
901#endif
902 /*
903 * fnsave initializes the FPU and destroys whatever
904 * context it contains. Make sure the FPU owner
905 * starts with a clean state next time.
906 */
907 npxdrop();
908 intr_restore(s);
909 return (_MC_FPOWNED_FPU);
910 } else {
911 intr_restore(s);
912 bcopy(&td->td_pcb->pcb_save, addr, sizeof(*addr));
913 return (_MC_FPOWNED_PCB);
914 }
915}
916
917/*
918 * Set the state of the FPU.
919 */
920void
921npxsetregs(td, addr)
922 struct thread *td;
923 union savefpu *addr;
924{
925 register_t s;
926
927 if (!npx_exists)
928 return;
929
930 s = intr_disable();
931 if (td == PCPU_GET(fpcurthread)) {
932#ifdef CPU_ENABLE_SSE
933 if (!cpu_fxsr)
934#endif
935 fnclex(); /* As in npxdrop(). */
936 fpurstor(addr);
937 intr_restore(s);
938 } else {
939 intr_restore(s);
940 bcopy(addr, &td->td_pcb->pcb_save, sizeof(*addr));
941 }
942 curthread->td_pcb->pcb_flags |= PCB_NPXINITDONE;
943}
944
|
980static void
981fpusave(addr)
982 union savefpu *addr;
983{
984
985#ifdef CPU_ENABLE_SSE
986 if (cpu_fxsr)
987 fxsave(addr);
988 else
989#endif
990 fnsave(addr);
991}
992
993#ifdef CPU_ENABLE_SSE
994/*
995 * On AuthenticAMD processors, the fxrstor instruction does not restore
996 * the x87's stored last instruction pointer, last data pointer, and last
997 * opcode values, except in the rare case in which the exception summary
998 * (ES) bit in the x87 status word is set to 1.
999 *
1000 * In order to avoid leaking this information across processes, we clean
1001 * these values by performing a dummy load before executing fxrstor().
1002 */
1003static double dummy_variable = 0.0;
1004static void
1005fpu_clean_state(void)
1006{
1007 u_short status;
1008
1009 /*
1010 * Clear the ES bit in the x87 status word if it is currently
1011 * set, in order to avoid causing a fault in the upcoming load.
1012 */
1013 fnstsw(&status);
1014 if (status & 0x80)
1015 fnclex();
1016
1017 /*
1018 * Load the dummy variable into the x87 stack. This mangles
1019 * the x87 stack, but we don't care since we're about to call
1020 * fxrstor() anyway.
1021 */
1022 __asm __volatile("ffree %%st(7); fld %0" : : "m" (dummy_variable));
1023}
1024#endif /* CPU_ENABLE_SSE */
1025
1026static void
1027fpurstor(addr)
1028 union savefpu *addr;
1029{
1030
1031#ifdef CPU_ENABLE_SSE
1032 if (cpu_fxsr) {
1033 fpu_clean_state();
1034 fxrstor(addr);
1035 } else
1036#endif
1037 frstor(addr);
1038}
1039
1040#ifdef I586_CPU_XXX
1041static long
1042timezero(funcname, func)
1043 const char *funcname;
1044 void (*func)(void *buf, size_t len);
1045
1046{
1047 void *buf;
1048#define BUFSIZE 1048576
1049 long usec;
1050 struct timeval finish, start;
1051
1052 buf = malloc(BUFSIZE, M_TEMP, M_NOWAIT);
1053 if (buf == NULL)
1054 return (BUFSIZE);
1055 microtime(&start);
1056 (*func)(buf, BUFSIZE);
1057 microtime(&finish);
1058 usec = 1000000 * (finish.tv_sec - start.tv_sec) +
1059 finish.tv_usec - start.tv_usec;
1060 if (usec <= 0)
1061 usec = 1;
1062 if (bootverbose)
1063 printf("%s bandwidth = %u kBps\n", funcname,
1064 (u_int32_t)(((BUFSIZE >> 10) * 1000000) / usec));
1065 free(buf, M_TEMP);
1066 return (usec);
1067}
1068#endif /* I586_CPU */
1069
1070static device_method_t npx_methods[] = {
1071 /* Device interface */
1072 DEVMETHOD(device_identify, npx_identify),
1073 DEVMETHOD(device_probe, npx_probe),
1074 DEVMETHOD(device_attach, npx_attach),
1075 DEVMETHOD(device_detach, bus_generic_detach),
1076 DEVMETHOD(device_shutdown, bus_generic_shutdown),
1077 DEVMETHOD(device_suspend, bus_generic_suspend),
1078 DEVMETHOD(device_resume, bus_generic_resume),
1079
1080 { 0, 0 }
1081};
1082
1083static driver_t npx_driver = {
1084 "npx",
1085 npx_methods,
1086 1, /* no softc */
1087};
1088
1089static devclass_t npx_devclass;
1090
1091/*
1092 * We prefer to attach to the root nexus so that the usual case (exception 16)
1093 * doesn't describe the processor as being `on isa'.
1094 */
1095DRIVER_MODULE(npx, nexus, npx_driver, npx_devclass, 0, 0);
1096
1097#ifdef DEV_ISA
1098/*
1099 * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI.
1100 */
1101static struct isa_pnp_id npxisa_ids[] = {
1102 { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */
1103 { 0 }
1104};
1105
1106static int
1107npxisa_probe(device_t dev)
1108{
1109 int result;
1110 if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) {
1111 device_quiet(dev);
1112 }
1113 return(result);
1114}
1115
1116static int
1117npxisa_attach(device_t dev)
1118{
1119 return (0);
1120}
1121
1122static device_method_t npxisa_methods[] = {
1123 /* Device interface */
1124 DEVMETHOD(device_probe, npxisa_probe),
1125 DEVMETHOD(device_attach, npxisa_attach),
1126 DEVMETHOD(device_detach, bus_generic_detach),
1127 DEVMETHOD(device_shutdown, bus_generic_shutdown),
1128 DEVMETHOD(device_suspend, bus_generic_suspend),
1129 DEVMETHOD(device_resume, bus_generic_resume),
1130
1131 { 0, 0 }
1132};
1133
1134static driver_t npxisa_driver = {
1135 "npxisa",
1136 npxisa_methods,
1137 1, /* no softc */
1138};
1139
1140static devclass_t npxisa_devclass;
1141
1142DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0);
1143#ifndef PC98
1144DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0);
1145#endif
1146#endif /* DEV_ISA */
| 945static void
946fpusave(addr)
947 union savefpu *addr;
948{
949
950#ifdef CPU_ENABLE_SSE
951 if (cpu_fxsr)
952 fxsave(addr);
953 else
954#endif
955 fnsave(addr);
956}
957
958#ifdef CPU_ENABLE_SSE
959/*
960 * On AuthenticAMD processors, the fxrstor instruction does not restore
961 * the x87's stored last instruction pointer, last data pointer, and last
962 * opcode values, except in the rare case in which the exception summary
963 * (ES) bit in the x87 status word is set to 1.
964 *
965 * In order to avoid leaking this information across processes, we clean
966 * these values by performing a dummy load before executing fxrstor().
967 */
968static double dummy_variable = 0.0;
969static void
970fpu_clean_state(void)
971{
972 u_short status;
973
974 /*
975 * Clear the ES bit in the x87 status word if it is currently
976 * set, in order to avoid causing a fault in the upcoming load.
977 */
978 fnstsw(&status);
979 if (status & 0x80)
980 fnclex();
981
982 /*
983 * Load the dummy variable into the x87 stack. This mangles
984 * the x87 stack, but we don't care since we're about to call
985 * fxrstor() anyway.
986 */
987 __asm __volatile("ffree %%st(7); fld %0" : : "m" (dummy_variable));
988}
989#endif /* CPU_ENABLE_SSE */
990
991static void
992fpurstor(addr)
993 union savefpu *addr;
994{
995
996#ifdef CPU_ENABLE_SSE
997 if (cpu_fxsr) {
998 fpu_clean_state();
999 fxrstor(addr);
1000 } else
1001#endif
1002 frstor(addr);
1003}
1004
1005#ifdef I586_CPU_XXX
1006static long
1007timezero(funcname, func)
1008 const char *funcname;
1009 void (*func)(void *buf, size_t len);
1010
1011{
1012 void *buf;
1013#define BUFSIZE 1048576
1014 long usec;
1015 struct timeval finish, start;
1016
1017 buf = malloc(BUFSIZE, M_TEMP, M_NOWAIT);
1018 if (buf == NULL)
1019 return (BUFSIZE);
1020 microtime(&start);
1021 (*func)(buf, BUFSIZE);
1022 microtime(&finish);
1023 usec = 1000000 * (finish.tv_sec - start.tv_sec) +
1024 finish.tv_usec - start.tv_usec;
1025 if (usec <= 0)
1026 usec = 1;
1027 if (bootverbose)
1028 printf("%s bandwidth = %u kBps\n", funcname,
1029 (u_int32_t)(((BUFSIZE >> 10) * 1000000) / usec));
1030 free(buf, M_TEMP);
1031 return (usec);
1032}
1033#endif /* I586_CPU */
1034
1035static device_method_t npx_methods[] = {
1036 /* Device interface */
1037 DEVMETHOD(device_identify, npx_identify),
1038 DEVMETHOD(device_probe, npx_probe),
1039 DEVMETHOD(device_attach, npx_attach),
1040 DEVMETHOD(device_detach, bus_generic_detach),
1041 DEVMETHOD(device_shutdown, bus_generic_shutdown),
1042 DEVMETHOD(device_suspend, bus_generic_suspend),
1043 DEVMETHOD(device_resume, bus_generic_resume),
1044
1045 { 0, 0 }
1046};
1047
1048static driver_t npx_driver = {
1049 "npx",
1050 npx_methods,
1051 1, /* no softc */
1052};
1053
1054static devclass_t npx_devclass;
1055
1056/*
1057 * We prefer to attach to the root nexus so that the usual case (exception 16)
1058 * doesn't describe the processor as being `on isa'.
1059 */
1060DRIVER_MODULE(npx, nexus, npx_driver, npx_devclass, 0, 0);
1061
1062#ifdef DEV_ISA
1063/*
1064 * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI.
1065 */
1066static struct isa_pnp_id npxisa_ids[] = {
1067 { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */
1068 { 0 }
1069};
1070
1071static int
1072npxisa_probe(device_t dev)
1073{
1074 int result;
1075 if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) {
1076 device_quiet(dev);
1077 }
1078 return(result);
1079}
1080
1081static int
1082npxisa_attach(device_t dev)
1083{
1084 return (0);
1085}
1086
1087static device_method_t npxisa_methods[] = {
1088 /* Device interface */
1089 DEVMETHOD(device_probe, npxisa_probe),
1090 DEVMETHOD(device_attach, npxisa_attach),
1091 DEVMETHOD(device_detach, bus_generic_detach),
1092 DEVMETHOD(device_shutdown, bus_generic_shutdown),
1093 DEVMETHOD(device_suspend, bus_generic_suspend),
1094 DEVMETHOD(device_resume, bus_generic_resume),
1095
1096 { 0, 0 }
1097};
1098
1099static driver_t npxisa_driver = {
1100 "npxisa",
1101 npxisa_methods,
1102 1, /* no softc */
1103};
1104
1105static devclass_t npxisa_devclass;
1106
1107DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0);
1108#ifndef PC98
1109DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0);
1110#endif
1111#endif /* DEV_ISA */
|