1/* 2 * linux/arch/i386/kernel/i387.c 3 * 4 * Copyright (C) 1994 Linus Torvalds 5 * 6 * Pentium III FXSR, SSE support 7 * General FPU state handling cleanups 8 * Gareth Hughes <gareth@valinux.com>, May 2000 9 */ 10 11#include <linux/sched.h> 12#include <linux/module.h> 13#include <asm/processor.h> 14#include <asm/i387.h> 15#include <asm/math_emu.h> 16#include <asm/sigcontext.h> 17#include <asm/user.h> 18#include <asm/ptrace.h> 19#include <asm/uaccess.h> 20 21#ifdef CONFIG_MATH_EMULATION 22#define HAVE_HWFP (boot_cpu_data.hard_math) 23#else 24#define HAVE_HWFP 1 25#endif 26 27static unsigned long mxcsr_feature_mask __read_mostly = 0xffffffff; 28 29void mxcsr_feature_mask_init(void) 30{ 31 unsigned long mask = 0; 32 clts(); 33 if (cpu_has_fxsr) { 34 memset(¤t->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); 35 asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); 36 mask = current->thread.i387.fxsave.mxcsr_mask; 37 if (mask == 0) mask = 0x0000ffbf; 38 } 39 mxcsr_feature_mask &= mask; 40 stts(); 41} 42 43/* 44 * The _current_ task is using the FPU for the first time 45 * so initialize it and set the mxcsr to its default 46 * value at reset if we support XMM instructions and then 47 * remeber the current task has used the FPU. 48 */ 49void init_fpu(struct task_struct *tsk) 50{ 51 if (cpu_has_fxsr) { 52 memset(&tsk->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); 53 tsk->thread.i387.fxsave.cwd = 0x37f; 54 if (cpu_has_xmm) 55 tsk->thread.i387.fxsave.mxcsr = 0x1f80; 56 } else { 57 memset(&tsk->thread.i387.fsave, 0, sizeof(struct i387_fsave_struct)); 58 tsk->thread.i387.fsave.cwd = 0xffff037fu; 59 tsk->thread.i387.fsave.swd = 0xffff0000u; 60 tsk->thread.i387.fsave.twd = 0xffffffffu; 61 tsk->thread.i387.fsave.fos = 0xffff0000u; 62 } 63 /* only the device not available exception or ptrace can call init_fpu */ 64 set_stopped_child_used_math(tsk); 65} 66 67/* 68 * FPU lazy state save handling. 69 */ 70 71void kernel_fpu_begin(void) 72{ 73 struct thread_info *thread = current_thread_info(); 74 75 preempt_disable(); 76 if (thread->status & TS_USEDFPU) { 77 __save_init_fpu(thread->task); 78 return; 79 } 80 clts(); 81} 82EXPORT_SYMBOL_GPL(kernel_fpu_begin); 83 84/* 85 * FPU tag word conversions. 86 */ 87 88static inline unsigned short twd_i387_to_fxsr( unsigned short twd ) 89{ 90 unsigned int tmp; /* to avoid 16 bit prefixes in the code */ 91 92 /* Transform each pair of bits into 01 (valid) or 00 (empty) */ 93 tmp = ~twd; 94 tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ 95 /* and move the valid bits to the lower byte. */ 96 tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ 97 tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ 98 tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ 99 return tmp; 100} 101 102static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave ) 103{ 104 struct _fpxreg *st = NULL; 105 unsigned long tos = (fxsave->swd >> 11) & 7; 106 unsigned long twd = (unsigned long) fxsave->twd; 107 unsigned long tag; 108 unsigned long ret = 0xffff0000u; 109 int i; 110 111#define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16); 112 113 for ( i = 0 ; i < 8 ; i++ ) { 114 if ( twd & 0x1 ) { 115 st = FPREG_ADDR( fxsave, (i - tos) & 7 ); 116 117 switch ( st->exponent & 0x7fff ) { 118 case 0x7fff: 119 tag = 2; /* Special */ 120 break; 121 case 0x0000: 122 if ( !st->significand[0] && 123 !st->significand[1] && 124 !st->significand[2] && 125 !st->significand[3] ) { 126 tag = 1; /* Zero */ 127 } else { 128 tag = 2; /* Special */ 129 } 130 break; 131 default: 132 if ( st->significand[3] & 0x8000 ) { 133 tag = 0; /* Valid */ 134 } else { 135 tag = 2; /* Special */ 136 } 137 break; 138 } 139 } else { 140 tag = 3; /* Empty */ 141 } 142 ret |= (tag << (2 * i)); 143 twd = twd >> 1; 144 } 145 return ret; 146} 147 148/* 149 * FPU state interaction. 150 */ 151 152unsigned short get_fpu_cwd( struct task_struct *tsk ) 153{ 154 if ( cpu_has_fxsr ) { 155 return tsk->thread.i387.fxsave.cwd; 156 } else { 157 return (unsigned short)tsk->thread.i387.fsave.cwd; 158 } 159} 160 161unsigned short get_fpu_swd( struct task_struct *tsk ) 162{ 163 if ( cpu_has_fxsr ) { 164 return tsk->thread.i387.fxsave.swd; 165 } else { 166 return (unsigned short)tsk->thread.i387.fsave.swd; 167 } 168} 169 170 171unsigned short get_fpu_mxcsr( struct task_struct *tsk ) 172{ 173 if ( cpu_has_xmm ) { 174 return tsk->thread.i387.fxsave.mxcsr; 175 } else { 176 return 0x1f80; 177 } 178} 179 180 181/* 182 * FXSR floating point environment conversions. 183 */ 184 185static int convert_fxsr_to_user( struct _fpstate __user *buf, 186 struct i387_fxsave_struct *fxsave ) 187{ 188 unsigned long env[7]; 189 struct _fpreg __user *to; 190 struct _fpxreg *from; 191 int i; 192 193 env[0] = (unsigned long)fxsave->cwd | 0xffff0000ul; 194 env[1] = (unsigned long)fxsave->swd | 0xffff0000ul; 195 env[2] = twd_fxsr_to_i387(fxsave); 196 env[3] = fxsave->fip; 197 env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16); 198 env[5] = fxsave->foo; 199 env[6] = fxsave->fos; 200 201 if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) ) 202 return 1; 203 204 to = &buf->_st[0]; 205 from = (struct _fpxreg *) &fxsave->st_space[0]; 206 for ( i = 0 ; i < 8 ; i++, to++, from++ ) { 207 unsigned long __user *t = (unsigned long __user *)to; 208 unsigned long *f = (unsigned long *)from; 209 210 if (__put_user(*f, t) || 211 __put_user(*(f + 1), t + 1) || 212 __put_user(from->exponent, &to->exponent)) 213 return 1; 214 } 215 return 0; 216} 217 218static int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave, 219 struct _fpstate __user *buf ) 220{ 221 unsigned long env[7]; 222 struct _fpxreg *to; 223 struct _fpreg __user *from; 224 int i; 225 226 if ( __copy_from_user( env, buf, 7 * sizeof(long) ) ) 227 return 1; 228 229 fxsave->cwd = (unsigned short)(env[0] & 0xffff); 230 fxsave->swd = (unsigned short)(env[1] & 0xffff); 231 fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff)); 232 fxsave->fip = env[3]; 233 fxsave->fop = (unsigned short)((env[4] & 0xffff0000ul) >> 16); 234 fxsave->fcs = (env[4] & 0xffff); 235 fxsave->foo = env[5]; 236 fxsave->fos = env[6]; 237 238 to = (struct _fpxreg *) &fxsave->st_space[0]; 239 from = &buf->_st[0]; 240 for ( i = 0 ; i < 8 ; i++, to++, from++ ) { 241 unsigned long *t = (unsigned long *)to; 242 unsigned long __user *f = (unsigned long __user *)from; 243 244 if (__get_user(*t, f) || 245 __get_user(*(t + 1), f + 1) || 246 __get_user(to->exponent, &from->exponent)) 247 return 1; 248 } 249 return 0; 250} 251 252/* 253 * Signal frame handlers. 254 */ 255 256static inline int save_i387_fsave( struct _fpstate __user *buf ) 257{ 258 struct task_struct *tsk = current; 259 260 unlazy_fpu( tsk ); 261 tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd; 262 if ( __copy_to_user( buf, &tsk->thread.i387.fsave, 263 sizeof(struct i387_fsave_struct) ) ) 264 return -1; 265 return 1; 266} 267 268static int save_i387_fxsave( struct _fpstate __user *buf ) 269{ 270 struct task_struct *tsk = current; 271 int err = 0; 272 273 unlazy_fpu( tsk ); 274 275 if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) ) 276 return -1; 277 278 err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status ); 279 err |= __put_user( X86_FXSR_MAGIC, &buf->magic ); 280 if ( err ) 281 return -1; 282 283 if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave, 284 sizeof(struct i387_fxsave_struct) ) ) 285 return -1; 286 return 1; 287} 288 289int save_i387( struct _fpstate __user *buf ) 290{ 291 if ( !used_math() ) 292 return 0; 293 294 /* This will cause a "finit" to be triggered by the next 295 * attempted FPU operation by the 'current' process. 296 */ 297 clear_used_math(); 298 299 if ( HAVE_HWFP ) { 300 if ( cpu_has_fxsr ) { 301 return save_i387_fxsave( buf ); 302 } else { 303 return save_i387_fsave( buf ); 304 } 305 } else { 306 return save_i387_soft( ¤t->thread.i387.soft, buf ); 307 } 308} 309 310static inline int restore_i387_fsave( struct _fpstate __user *buf ) 311{ 312 struct task_struct *tsk = current; 313 clear_fpu( tsk ); 314 return __copy_from_user( &tsk->thread.i387.fsave, buf, 315 sizeof(struct i387_fsave_struct) ); 316} 317 318static int restore_i387_fxsave( struct _fpstate __user *buf ) 319{ 320 int err; 321 struct task_struct *tsk = current; 322 clear_fpu( tsk ); 323 err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0], 324 sizeof(struct i387_fxsave_struct) ); 325 /* mxcsr reserved bits must be masked to zero for security reasons */ 326 tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; 327 return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf ); 328} 329 330int restore_i387( struct _fpstate __user *buf ) 331{ 332 int err; 333 334 if ( HAVE_HWFP ) { 335 if ( cpu_has_fxsr ) { 336 err = restore_i387_fxsave( buf ); 337 } else { 338 err = restore_i387_fsave( buf ); 339 } 340 } else { 341 err = restore_i387_soft( ¤t->thread.i387.soft, buf ); 342 } 343 set_used_math(); 344 return err; 345} 346 347/* 348 * ptrace request handlers. 349 */ 350 351static inline int get_fpregs_fsave( struct user_i387_struct __user *buf, 352 struct task_struct *tsk ) 353{ 354 return __copy_to_user( buf, &tsk->thread.i387.fsave, 355 sizeof(struct user_i387_struct) ); 356} 357 358static inline int get_fpregs_fxsave( struct user_i387_struct __user *buf, 359 struct task_struct *tsk ) 360{ 361 return convert_fxsr_to_user( (struct _fpstate __user *)buf, 362 &tsk->thread.i387.fxsave ); 363} 364 365int get_fpregs( struct user_i387_struct __user *buf, struct task_struct *tsk ) 366{ 367 if ( HAVE_HWFP ) { 368 if ( cpu_has_fxsr ) { 369 return get_fpregs_fxsave( buf, tsk ); 370 } else { 371 return get_fpregs_fsave( buf, tsk ); 372 } 373 } else { 374 return save_i387_soft( &tsk->thread.i387.soft, 375 (struct _fpstate __user *)buf ); 376 } 377} 378 379static inline int set_fpregs_fsave( struct task_struct *tsk, 380 struct user_i387_struct __user *buf ) 381{ 382 return __copy_from_user( &tsk->thread.i387.fsave, buf, 383 sizeof(struct user_i387_struct) ); 384} 385 386static inline int set_fpregs_fxsave( struct task_struct *tsk, 387 struct user_i387_struct __user *buf ) 388{ 389 return convert_fxsr_from_user( &tsk->thread.i387.fxsave, 390 (struct _fpstate __user *)buf ); 391} 392 393int set_fpregs( struct task_struct *tsk, struct user_i387_struct __user *buf ) 394{ 395 if ( HAVE_HWFP ) { 396 if ( cpu_has_fxsr ) { 397 return set_fpregs_fxsave( tsk, buf ); 398 } else { 399 return set_fpregs_fsave( tsk, buf ); 400 } 401 } else { 402 return restore_i387_soft( &tsk->thread.i387.soft, 403 (struct _fpstate __user *)buf ); 404 } 405} 406 407int get_fpxregs( struct user_fxsr_struct __user *buf, struct task_struct *tsk ) 408{ 409 if ( cpu_has_fxsr ) { 410 if (__copy_to_user( buf, &tsk->thread.i387.fxsave, 411 sizeof(struct user_fxsr_struct) )) 412 return -EFAULT; 413 return 0; 414 } else { 415 return -EIO; 416 } 417} 418 419int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct __user *buf ) 420{ 421 int ret = 0; 422 423 if ( cpu_has_fxsr ) { 424 if (__copy_from_user( &tsk->thread.i387.fxsave, buf, 425 sizeof(struct user_fxsr_struct) )) 426 ret = -EFAULT; 427 /* mxcsr reserved bits must be masked to zero for security reasons */ 428 tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; 429 } else { 430 ret = -EIO; 431 } 432 return ret; 433} 434 435/* 436 * FPU state for core dumps. 437 */ 438 439static inline void copy_fpu_fsave( struct task_struct *tsk, 440 struct user_i387_struct *fpu ) 441{ 442 memcpy( fpu, &tsk->thread.i387.fsave, 443 sizeof(struct user_i387_struct) ); 444} 445 446static inline void copy_fpu_fxsave( struct task_struct *tsk, 447 struct user_i387_struct *fpu ) 448{ 449 unsigned short *to; 450 unsigned short *from; 451 int i; 452 453 memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) ); 454 455 to = (unsigned short *)&fpu->st_space[0]; 456 from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0]; 457 for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) { 458 memcpy( to, from, 5 * sizeof(unsigned short) ); 459 } 460} 461 462int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu ) 463{ 464 int fpvalid; 465 struct task_struct *tsk = current; 466 467 fpvalid = !!used_math(); 468 if ( fpvalid ) { 469 unlazy_fpu( tsk ); 470 if ( cpu_has_fxsr ) { 471 copy_fpu_fxsave( tsk, fpu ); 472 } else { 473 copy_fpu_fsave( tsk, fpu ); 474 } 475 } 476 477 return fpvalid; 478} 479EXPORT_SYMBOL(dump_fpu); 480 481int dump_task_fpu(struct task_struct *tsk, struct user_i387_struct *fpu) 482{ 483 int fpvalid = !!tsk_used_math(tsk); 484 485 if (fpvalid) { 486 if (tsk == current) 487 unlazy_fpu(tsk); 488 if (cpu_has_fxsr) 489 copy_fpu_fxsave(tsk, fpu); 490 else 491 copy_fpu_fsave(tsk, fpu); 492 } 493 return fpvalid; 494} 495 496int dump_task_extended_fpu(struct task_struct *tsk, struct user_fxsr_struct *fpu) 497{ 498 int fpvalid = tsk_used_math(tsk) && cpu_has_fxsr; 499 500 if (fpvalid) { 501 if (tsk == current) 502 unlazy_fpu(tsk); 503 memcpy(fpu, &tsk->thread.i387.fxsave, sizeof(*fpu)); 504 } 505 return fpvalid; 506} 507