arch/i386/kernel/i387.c

   1 /*
   2  *  linux/arch/i386/kernel/i387.c
   3  *
   4  *  Copyright (C) 1994 Linus Torvalds
   5  *
   6  *  Pentium III FXSR, SSE support
   7  *  General FPU state handling cleanups
   8  *      Gareth Hughes <gareth@valinux.com>, May 2000
   9  */
  10
  11 #include <linux/config.h>
  12 #include <linux/sched.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel_stat.h>
  15 #include <asm/processor.h>
  16 #include <asm/i387.h>
  17 #include <asm/math_emu.h>
  18 #include <asm/sigcontext.h>
  19 #include <asm/user.h>
  20 #include <asm/ptrace.h>
  21 #include <asm/uaccess.h>
  22
  23 #ifdef CONFIG_MATH_EMULATION
  24 #define HAVE_HWFP (boot_cpu_data.hard_math)
  25 #else
  26 #define HAVE_HWFP 1
  27 #endif
  28
  29 static union i387_union empty_fpu_state;
  30
  31 void __init boot_init_fpu(void)
  32 {
  33         memset(&empty_fpu_state, 0, sizeof(union i387_union));
  34
  35         if (!cpu_has_fxsr) {
  36                 empty_fpu_state.fsave.cwd = 0xffff037f;
  37                 empty_fpu_state.fsave.swd = 0xffff0000;
  38                 empty_fpu_state.fsave.twd = 0xffffffff;
  39                 empty_fpu_state.fsave.fos = 0xffff0000;
  40         } else {
  41                 empty_fpu_state.fxsave.cwd = 0x37f;
  42                 if (cpu_has_xmm)
  43                         empty_fpu_state.fxsave.mxcsr = 0x1f80;
  44         }
  45 }
  46
  47 void load_empty_fpu(struct task_struct * tsk)
  48 {
  49         memcpy(&tsk->thread.i387, &empty_fpu_state, sizeof(union i387_union));
  50 }
  51
  52 /*
  53  * The _current_ task is using the FPU for the first time
  54  * so initialize it and set the mxcsr to its default
  55  * value at reset if we support XMM instructions and then
  56  * remeber the current task has used the FPU.
  57  */
  58 void init_fpu(void)
  59 {
  60         if (cpu_has_fxsr)
  61                 asm volatile("fxrstor %0" : : "m" (empty_fpu_state.fxsave));
  62         else
  63                 __asm__("fninit");
  64         current->used_math = 1;
  65 }
  66
  67 /*
  68  * FPU lazy state save handling.
  69  */
  70
  71 static inline void __save_init_fpu( struct task_struct *tsk )
  72 {
  73         if ( cpu_has_fxsr ) {
  74                 asm volatile( "fxsave %0"
  75                               : "=m" (tsk->thread.i387.fxsave) );
  76                 if (tsk->thread.i387.fxsave.swd & (1<<7))
  77                         asm volatile("fnclex");
  78                 /* AMD CPUs leak F?P. Clear it here */
  79                 asm volatile("ffree %%st(7) ; fildl %0" :: "m" (kstat.context_swtch));
  80         } else {
  81                 asm volatile( "fnsave %0 ; fwait"
  82                               : "=m" (tsk->thread.i387.fsave) );
  83         }
  84         tsk->flags &= ~PF_USEDFPU;
  85 }
  86
  87 void save_init_fpu( struct task_struct *tsk )
  88 {
  89         __save_init_fpu(tsk);
  90         stts();
  91 }
  92
  93 void kernel_fpu_begin(void)
  94 {
  95         struct task_struct *tsk = current;
  96
  97         if (tsk->flags & PF_USEDFPU) {
  98                 __save_init_fpu(tsk);
  99                 return;
 100         }
 101         clts();
 102 }
 103
 104 void restore_fpu( struct task_struct *tsk )
 105 {
 106         if ( cpu_has_fxsr ) {
 107                 asm volatile( "fxrstor %0"
 108                               : : "m" (tsk->thread.i387.fxsave) );
 109         } else {
 110                 asm volatile( "frstor %0"
 111                               : : "m" (tsk->thread.i387.fsave) );
 112         }
 113 }
 114
 115 /*
 116  * FPU tag word conversions.
 117  */
 118
 119 static inline unsigned short twd_i387_to_fxsr( unsigned short twd )
 120 {
 121         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
 122
 123         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
 124         tmp = ~twd;
 125         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
 126         /* and move the valid bits to the lower byte. */
 127         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
 128         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
 129         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
 130         return tmp;
 131 }
 132
 133 static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave )
 134 {
 135         struct _fpxreg *st = NULL;
 136         unsigned long tos = (fxsave->swd >> 11) & 7;
 137         unsigned long twd = (unsigned long) fxsave->twd;
 138         unsigned long tag;
 139         unsigned long ret = 0xffff0000;
 140         int i;
 141
 142 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16);
 143
 144         for ( i = 0 ; i < 8 ; i++ ) {
 145                 if ( twd & 0x1 ) {
 146                         st = FPREG_ADDR( fxsave, (i - tos) & 7 );
 147
 148                         switch ( st->exponent & 0x7fff ) {
 149                         case 0x7fff:
 150                                 tag = 2;                /* Special */
 151                                 break;
 152                         case 0x0000:
 153                                 if ( !st->significand[0] &&
 154                                      !st->significand[1] &&
 155                                      !st->significand[2] &&
 156                                      !st->significand[3] ) {
 157                                         tag = 1;        /* Zero */
 158                                 } else {
 159                                         tag = 2;        /* Special */
 160                                 }
 161                                 break;
 162                         default:
 163                                 if ( st->significand[3] & 0x8000 ) {
 164                                         tag = 0;        /* Valid */
 165                                 } else {
 166                                         tag = 2;        /* Special */
 167                                 }
 168                                 break;
 169                         }
 170                 } else {
 171                         tag = 3;                        /* Empty */
 172                 }
 173                 ret |= (tag << (2 * i));
 174                 twd = twd >> 1;
 175         }
 176         return ret;
 177 }
 178
 179 /*
 180  * FPU state interaction.
 181  */
 182
 183 unsigned short get_fpu_cwd( struct task_struct *tsk )
 184 {
 185         if ( cpu_has_fxsr ) {
 186                 return tsk->thread.i387.fxsave.cwd;
 187         } else {
 188                 return (unsigned short)tsk->thread.i387.fsave.cwd;
 189         }
 190 }
 191
 192 unsigned short get_fpu_swd( struct task_struct *tsk )
 193 {
 194         if ( cpu_has_fxsr ) {
 195                 return tsk->thread.i387.fxsave.swd;
 196         } else {
 197                 return (unsigned short)tsk->thread.i387.fsave.swd;
 198         }
 199 }
 200
 201 unsigned short get_fpu_twd( struct task_struct *tsk )
 202 {
 203         if ( cpu_has_fxsr ) {
 204                 return tsk->thread.i387.fxsave.twd;
 205         } else {
 206                 return (unsigned short)tsk->thread.i387.fsave.twd;
 207         }
 208 }
 209
 210 unsigned short get_fpu_mxcsr( struct task_struct *tsk )
 211 {
 212         if ( cpu_has_xmm ) {
 213                 return tsk->thread.i387.fxsave.mxcsr;
 214         } else {
 215                 return 0x1f80;
 216         }
 217 }
 218
 219 void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd )
 220 {
 221         if ( cpu_has_fxsr ) {
 222                 tsk->thread.i387.fxsave.cwd = cwd;
 223         } else {
 224                 tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000);
 225         }
 226 }
 227
 228 void set_fpu_swd( struct task_struct *tsk, unsigned short swd )
 229 {
 230         if ( cpu_has_fxsr ) {
 231                 tsk->thread.i387.fxsave.swd = swd;
 232         } else {
 233                 tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000);
 234         }
 235 }
 236
 237 void set_fpu_twd( struct task_struct *tsk, unsigned short twd )
 238 {
 239         if ( cpu_has_fxsr ) {
 240                 tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd);
 241         } else {
 242                 tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000);
 243         }
 244 }
 245
 246 void set_fpu_mxcsr( struct task_struct *tsk, unsigned short mxcsr )
 247 {
 248         if ( cpu_has_xmm ) {
 249                 tsk->thread.i387.fxsave.mxcsr = (mxcsr & 0xffbf);
 250         }
 251 }
 252
 253 /*
 254  * FXSR floating point environment conversions.
 255  */
 256
 257 static inline int convert_fxsr_to_user( struct _fpstate *buf,
 258                                         struct i387_fxsave_struct *fxsave )
 259 {
 260         unsigned long env[7];
 261         struct _fpreg *to;
 262         struct _fpxreg *from;
 263         int i;
 264
 265         env[0] = (unsigned long)fxsave->cwd | 0xffff0000;
 266         env[1] = (unsigned long)fxsave->swd | 0xffff0000;
 267         env[2] = twd_fxsr_to_i387(fxsave);
 268         env[3] = fxsave->fip;
 269         env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16);
 270         env[5] = fxsave->foo;
 271         env[6] = fxsave->fos;
 272
 273         if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) )
 274                 return 1;
 275
 276         to = &buf->_st[0];
 277         from = (struct _fpxreg *) &fxsave->st_space[0];
 278         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 279                 if ( __copy_to_user( to, from, sizeof(*to) ) )
 280                         return 1;
 281         }
 282         return 0;
 283 }
 284
 285 static inline int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave,
 286                                           struct _fpstate *buf )
 287 {
 288         unsigned long env[7];
 289         struct _fpxreg *to;
 290         struct _fpreg *from;
 291         int i;
 292
 293         if ( __copy_from_user( env, buf, 7 * sizeof(long) ) )
 294                 return 1;
 295
 296         fxsave->cwd = (unsigned short)(env[0] & 0xffff);
 297         fxsave->swd = (unsigned short)(env[1] & 0xffff);
 298         fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff));
 299         fxsave->fip = env[3];
 300         fxsave->fop = (unsigned short)((env[4] & 0xffff0000) >> 16);
 301         fxsave->fcs = (env[4] & 0xffff);
 302         fxsave->foo = env[5];
 303         fxsave->fos = env[6];
 304
 305         to = (struct _fpxreg *) &fxsave->st_space[0];
 306         from = &buf->_st[0];
 307         for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
 308                 if ( __copy_from_user( to, from, sizeof(*from) ) )
 309                         return 1;
 310         }
 311         return 0;
 312 }
 313
 314 /*
 315  * Signal frame handlers.
 316  */
 317
 318 static inline int save_i387_fsave( struct _fpstate *buf )
 319 {
 320         struct task_struct *tsk = current;
 321
 322         unlazy_fpu( tsk );
 323         tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
 324         if ( __copy_to_user( buf, &tsk->thread.i387.fsave,
 325                              sizeof(struct i387_fsave_struct) ) )
 326                 return -1;
 327         return 1;
 328 }
 329
 330 static inline int save_i387_fxsave( struct _fpstate *buf )
 331 {
 332         struct task_struct *tsk = current;
 333         int err = 0;
 334
 335         unlazy_fpu( tsk );
 336
 337         if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) )
 338                 return -1;
 339
 340         err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status );
 341         err |= __put_user( X86_FXSR_MAGIC, &buf->magic );
 342         if ( err )
 343                 return -1;
 344
 345         if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
 346                              sizeof(struct i387_fxsave_struct) ) )
 347                 return -1;
 348         return 1;
 349 }
 350
 351 int save_i387( struct _fpstate *buf )
 352 {
 353         if ( !current->used_math )
 354                 return 0;
 355
 356         /* This will cause a "finit" to be triggered by the next
 357          * attempted FPU operation by the 'current' process.
 358          */
 359         current->used_math = 0;
 360
 361         if ( HAVE_HWFP ) {
 362                 if ( cpu_has_fxsr ) {
 363                         return save_i387_fxsave( buf );
 364                 } else {
 365                         return save_i387_fsave( buf );
 366                 }
 367         } else {
 368                 return save_i387_soft( &current->thread.i387.soft, buf );
 369         }
 370 }
 371
 372 static inline int restore_i387_fsave( struct _fpstate *buf )
 373 {
 374         struct task_struct *tsk = current;
 375         clear_fpu( tsk );
 376         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 377                                  sizeof(struct i387_fsave_struct) );
 378 }
 379
 380 static inline int restore_i387_fxsave( struct _fpstate *buf )
 381 {
 382         int err;
 383         struct task_struct *tsk = current;
 384         clear_fpu( tsk );
 385         err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
 386                                 sizeof(struct i387_fxsave_struct) );
 387         /* mxcsr bit 6 and 31-16 must be zero for security reasons */
 388         tsk->thread.i387.fxsave.mxcsr &= 0xffbf;
 389         return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf );
 390 }
 391
 392 int restore_i387( struct _fpstate *buf )
 393 {
 394         int err;
 395
 396         if ( HAVE_HWFP ) {
 397                 if ( cpu_has_fxsr ) {
 398                         err =  restore_i387_fxsave( buf );
 399                 } else {
 400                         err = restore_i387_fsave( buf );
 401                 }
 402         } else {
 403                 err = restore_i387_soft( &current->thread.i387.soft, buf );
 404         }
 405         current->used_math = 1;
 406         return err;
 407 }
 408
 409 /*
 410  * ptrace request handlers.
 411  */
 412
 413 static inline int get_fpregs_fsave( struct user_i387_struct *buf,
 414                                     struct task_struct *tsk )
 415 {
 416         return __copy_to_user( buf, &tsk->thread.i387.fsave,
 417                                sizeof(struct user_i387_struct) );
 418 }
 419
 420 static inline int get_fpregs_fxsave( struct user_i387_struct *buf,
 421                                      struct task_struct *tsk )
 422 {
 423         return convert_fxsr_to_user( (struct _fpstate *)buf,
 424                                      &tsk->thread.i387.fxsave );
 425 }
 426
 427 int get_fpregs( struct user_i387_struct *buf, struct task_struct *tsk )
 428 {
 429         if ( HAVE_HWFP ) {
 430                 if ( cpu_has_fxsr ) {
 431                         return get_fpregs_fxsave( buf, tsk );
 432                 } else {
 433                         return get_fpregs_fsave( buf, tsk );
 434                 }
 435         } else {
 436                 return save_i387_soft( &tsk->thread.i387.soft,
 437                                        (struct _fpstate *)buf );
 438         }
 439 }
 440
 441 static inline int set_fpregs_fsave( struct task_struct *tsk,
 442                                     struct user_i387_struct *buf )
 443 {
 444         return __copy_from_user( &tsk->thread.i387.fsave, buf,
 445                                  sizeof(struct user_i387_struct) );
 446 }
 447
 448 static inline int set_fpregs_fxsave( struct task_struct *tsk,
 449                                      struct user_i387_struct *buf )
 450 {
 451         return convert_fxsr_from_user( &tsk->thread.i387.fxsave,
 452                                        (struct _fpstate *)buf );
 453 }
 454
 455 int set_fpregs( struct task_struct *tsk, struct user_i387_struct *buf )
 456 {
 457         if ( HAVE_HWFP ) {
 458                 if ( cpu_has_fxsr ) {
 459                         return set_fpregs_fxsave( tsk, buf );
 460                 } else {
 461                         return set_fpregs_fsave( tsk, buf );
 462                 }
 463         } else {
 464                 return restore_i387_soft( &tsk->thread.i387.soft,
 465                                           (struct _fpstate *)buf );
 466         }
 467 }
 468
 469 int get_fpxregs( struct user_fxsr_struct *buf, struct task_struct *tsk )
 470 {
 471         if ( cpu_has_fxsr ) {
 472                 if (__copy_to_user( (void *)buf, &tsk->thread.i387.fxsave,
 473                                     sizeof(struct user_fxsr_struct) ))
 474                         return -EFAULT;
 475                 return 0;
 476         } else {
 477                 return -EIO;
 478         }
 479 }
 480
 481 int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct *buf )
 482 {
 483         if ( cpu_has_fxsr ) {
 484                 __copy_from_user( &tsk->thread.i387.fxsave, (void *)buf,
 485                                   sizeof(struct user_fxsr_struct) );
 486                 /* mxcsr bit 6 and 31-16 must be zero for security reasons */
 487                 tsk->thread.i387.fxsave.mxcsr &= 0xffbf;
 488                 return 0;
 489         } else {
 490                 return -EIO;
 491         }
 492 }
 493
 494 /*
 495  * FPU state for core dumps.
 496  */
 497
 498 static inline void copy_fpu_fsave( struct task_struct *tsk,
 499                                    struct user_i387_struct *fpu )
 500 {
 501         memcpy( fpu, &tsk->thread.i387.fsave,
 502                 sizeof(struct user_i387_struct) );
 503 }
 504
 505 static inline void copy_fpu_fxsave( struct task_struct *tsk,
 506                                    struct user_i387_struct *fpu )
 507 {
 508         unsigned short *to;
 509         unsigned short *from;
 510         int i;
 511
 512         memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) );
 513
 514         to = (unsigned short *)&fpu->st_space[0];
 515         from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0];
 516         for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) {
 517                 memcpy( to, from, 5 * sizeof(unsigned short) );
 518         }
 519 }
 520
 521 int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu )
 522 {
 523         int fpvalid;
 524         struct task_struct *tsk = current;
 525
 526         fpvalid = tsk->used_math;
 527         if ( fpvalid ) {
 528                 unlazy_fpu( tsk );
 529                 if ( cpu_has_fxsr ) {
 530                         copy_fpu_fxsave( tsk, fpu );
 531                 } else {
 532                         copy_fpu_fsave( tsk, fpu );
 533                 }
 534         }
 535
 536         return fpvalid;
 537 }
 538
 539 int dump_extended_fpu( struct pt_regs *regs, struct user_fxsr_struct *fpu )
 540 {
 541         int fpvalid;
 542         struct task_struct *tsk = current;
 543
 544         fpvalid = tsk->used_math && cpu_has_fxsr;
 545         if ( fpvalid ) {
 546                 unlazy_fpu( tsk );
 547                 memcpy( fpu, &tsk->thread.i387.fxsave,
 548                         sizeof(struct user_fxsr_struct) );
 549         }
 550
 551         return fpvalid;
 552 }