reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The TSS to use for Double Fault Traps (INT 0x9) */
  18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  19
  20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  21 UCHAR KiNMITSS[KTSS_IO_MAPS];
  22
  23 /* CPU Features and Flags */
  24 ULONG KeI386CpuType;
  25 ULONG KeI386CpuStep;
  26 ULONG KeProcessorArchitecture;
  27 ULONG KeProcessorLevel;
  28 ULONG KeProcessorRevision;
  29 ULONG KeFeatureBits;
  30 ULONG KiFastSystemCallDisable;
  31 ULONG KeI386NpxPresent = 0;
  32 ULONG KiMXCsrMask = 0;
  33 ULONG MxcsrFeatureMask = 0;
  34 ULONG KeI386XMMIPresent = 0;
  35 ULONG KeI386FxsrPresent = 0;
  36 ULONG KeI386MachineType;
  37 ULONG Ke386Pae = FALSE;
  38 ULONG Ke386NoExecute = FALSE;
  39 ULONG KeLargestCacheLine = 0x40;
  40 ULONG KeDcacheFlushCount = 0;
  41 ULONG KeIcacheFlushCount = 0;
  42 ULONG KiDmaIoCoherency = 0;
  43 ULONG KePrefetchNTAGranularity = 32;
  44 CHAR KeNumberProcessors;
  45 KAFFINITY KeActiveProcessors = 1;
  46 BOOLEAN KiI386PentiumLockErrataPresent;
  47 BOOLEAN KiSMTProcessorsPresent;
  48
  49 /* The distance between SYSEXIT and IRETD return modes */
  50 UCHAR KiSystemCallExitAdjust;
  51
  52 /* The offset that was applied -- either 0 or the value above */
  53 UCHAR KiSystemCallExitAdjusted;
  54
  55 /* Whether the adjustment was already done once */
  56 BOOLEAN KiFastCallCopyDoneOnce;
  57
  58 /* Flush data */
  59 volatile LONG KiTbFlushTimeStamp;
  60
  61 /* CPU Signatures */
  62 static const CHAR CmpIntelID[]       = "GenuineIntel";
  63 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  64 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  65 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  66 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  67 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  68
  69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  70
  71 VOID
  72 NTAPI
  73 CPUID(IN ULONG InfoType,
  74       OUT PULONG CpuInfoEax,
  75       OUT PULONG CpuInfoEbx,
  76       OUT PULONG CpuInfoEcx,
  77       OUT PULONG CpuInfoEdx)
  78 {
  79     ULONG CpuInfo[4];
  80
  81     /* Perform the CPUID Operation */
  82     __cpuid((int*)CpuInfo, InfoType);
  83
  84     /* Return the results */
  85     *CpuInfoEax = CpuInfo[0];
  86     *CpuInfoEbx = CpuInfo[1];
  87     *CpuInfoEcx = CpuInfo[2];
  88     *CpuInfoEdx = CpuInfo[3];
  89 }
  90
  91 VOID
  92 NTAPI
  93 WRMSR(IN ULONG Register,
  94       IN LONGLONG Value)
  95 {
  96    /* Write to the MSR */
  97     __writemsr(Register, Value);
  98 }
  99
 100 LONGLONG
 101 FASTCALL
 102 RDMSR(IN ULONG Register)
 103 {
 104     /* Read from the MSR */
 105     return __readmsr(Register);
 106 }
 107
 108 /* FUNCTIONS *****************************************************************/
 109
 110 VOID
 111 NTAPI
 112 KiSetProcessorType(VOID)
 113 {
 114     ULONG EFlags, NewEFlags;
 115     ULONG Reg, Dummy;
 116     ULONG Stepping, Type;
 117
 118     /* Start by assuming no CPUID data */
 119     KeGetCurrentPrcb()->CpuID = 0;
 120
 121     /* Save EFlags */
 122     EFlags = __readeflags();
 123
 124     /* XOR out the ID bit and update EFlags */
 125     NewEFlags = EFlags ^ EFLAGS_ID;
 126     __writeeflags(NewEFlags);
 127
 128     /* Get them back and see if they were modified */
 129     NewEFlags = __readeflags();
 130     if (NewEFlags != EFlags)
 131     {
 132         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 133         EFlags |= EFLAGS_ID;
 134         __writeeflags(EFlags);
 135
 136         /* Peform CPUID 0 to see if CPUID 1 is supported */
 137         CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
 138         if (Reg > 0)
 139         {
 140             /* Do CPUID 1 now */
 141             CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
 142
 143             /*
 144              * Get the Stepping and Type. The stepping contains both the
 145              * Model and the Step, while the Type contains the returned Type.
 146              * We ignore the family.
 147              *
 148              * For the stepping, we convert this: zzzzzzxy into this: x0y
 149              */
 150             Stepping = Reg & 0xF0;
 151             Stepping <<= 4;
 152             Stepping += (Reg & 0xFF);
 153             Stepping &= 0xF0F;
 154             Type = Reg & 0xF00;
 155             Type >>= 8;
 156
 157             /* Save them in the PRCB */
 158             KeGetCurrentPrcb()->CpuID = TRUE;
 159             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 160             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 161         }
 162         else
 163         {
 164             DPRINT1("CPUID Support lacking\n");
 165         }
 166     }
 167     else
 168     {
 169         DPRINT1("CPUID Support lacking\n");
 170     }
 171
 172     /* Restore EFLAGS */
 173     __writeeflags(EFlags);
 174 }
 175
 176 ULONG
 177 NTAPI
 178 KiGetCpuVendor(VOID)
 179 {
 180     PKPRCB Prcb = KeGetCurrentPrcb();
 181     ULONG Vendor[5];
 182     ULONG Temp;
 183
 184     /* Assume no Vendor ID and fail if no CPUID Support. */
 185     Prcb->VendorString[0] = 0;
 186     if (!Prcb->CpuID) return 0;
 187
 188     /* Get the Vendor ID and null-terminate it */
 189     CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
 190     Vendor[4] = 0;
 191
 192     /* Re-arrange vendor string */
 193     Temp = Vendor[2];
 194     Vendor[2] = Vendor[3];
 195     Vendor[3] = Temp;
 196
 197     /* Copy it to the PRCB and null-terminate it again */
 198     RtlCopyMemory(Prcb->VendorString,
 199                   &Vendor[1],
 200                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 201     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 202
 203     /* Now check the CPU Type */
 204     if (!strcmp(Prcb->VendorString, CmpIntelID))
 205     {
 206         return CPU_INTEL;
 207     }
 208     else if (!strcmp(Prcb->VendorString, CmpAmdID))
 209     {
 210         return CPU_AMD;
 211     }
 212     else if (!strcmp(Prcb->VendorString, CmpCyrixID))
 213     {
 214         DPRINT1("Cyrix CPU support not fully tested!\n");
 215         return CPU_CYRIX;
 216     }
 217     else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
 218     {
 219         DPRINT1("Transmeta CPU support not fully tested!\n");
 220         return CPU_TRANSMETA;
 221     }
 222     else if (!strcmp(Prcb->VendorString, CmpCentaurID))
 223     {
 224         DPRINT1("Centaur CPU support not fully tested!\n");
 225         return CPU_CENTAUR;
 226     }
 227     else if (!strcmp(Prcb->VendorString, CmpRiseID))
 228     {
 229         DPRINT1("Rise CPU support not fully tested!\n");
 230         return CPU_RISE;
 231     }
 232
 233     /* Invalid CPU */
 234     return 0;
 235 }
 236
 237 ULONG
 238 NTAPI
 239 KiGetFeatureBits(VOID)
 240 {
 241     PKPRCB Prcb = KeGetCurrentPrcb();
 242     ULONG Vendor;
 243     ULONG FeatureBits = KF_WORKING_PTE;
 244     ULONG Reg[4], Dummy;
 245     BOOLEAN ExtendedCPUID = TRUE;
 246     ULONG CpuFeatures = 0;
 247
 248     /* Get the Vendor ID */
 249     Vendor = KiGetCpuVendor();
 250
 251     /* Make sure we got a valid vendor ID at least. */
 252     if (!Vendor) return FeatureBits;
 253
 254     /* Get the CPUID Info. Features are in Reg[3]. */
 255     CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
 256
 257     /* Set the initial APIC ID */
 258     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 259
 260     switch (Vendor)
 261     {
 262         /* Intel CPUs */
 263         case CPU_INTEL:
 264
 265             /* Check if it's a P6 */
 266             if (Prcb->CpuType == 6)
 267             {
 268                 /* Perform the special sequence to get the MicroCode Signature */
 269                 WRMSR(0x8B, 0);
 270                 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
 271                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 272             }
 273             else if (Prcb->CpuType == 5)
 274             {
 275                 /* On P5, enable workaround for the LOCK errata. */
 276                 KiI386PentiumLockErrataPresent = TRUE;
 277             }
 278
 279             /* Check for broken P6 with bad SMP PTE implementation */
 280             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 281                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 282             {
 283                 /* Remove support for correct PTE support. */
 284                 FeatureBits &= ~KF_WORKING_PTE;
 285             }
 286
 287             /* Check if the CPU is too old to support SYSENTER */
 288             if ((Prcb->CpuType < 6) ||
 289                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 290             {
 291                 /* Disable it */
 292                 Reg[3] &= ~0x800;
 293             }
 294
 295             /* Set the current features */
 296             CpuFeatures = Reg[3];
 297
 298             break;
 299
 300         /* AMD CPUs */
 301         case CPU_AMD:
 302
 303             /* Check if this is a K5 or K6. (family 5) */
 304             if ((Reg[0] & 0x0F00) == 0x0500)
 305             {
 306                 /* Get the Model Number */
 307                 switch (Reg[0] & 0x00F0)
 308                 {
 309                     /* Model 1: K5 - 5k86 (initial models) */
 310                     case 0x0010:
 311
 312                         /* Check if this is Step 0 or 1. They don't support PGE */
 313                         if ((Reg[0] & 0x000F) > 0x03) break;
 314
 315                     /* Model 0: K5 - SSA5 */
 316                     case 0x0000:
 317
 318                         /* Model 0 doesn't support PGE at all. */
 319                         Reg[3] &= ~0x2000;
 320                         break;
 321
 322                     /* Model 8: K6-2 */
 323                     case 0x0080:
 324
 325                         /* K6-2, Step 8 and over have support for MTRR. */
 326                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 327                         break;
 328
 329                     /* Model 9: K6-III
 330                        Model D: K6-2+, K6-III+ */
 331                     case 0x0090:
 332                     case 0x00D0:
 333
 334                         FeatureBits |= KF_AMDK6MTRR;
 335                         break;
 336                 }
 337             }
 338             else if((Reg[0] & 0x0F00) < 0x0500)
 339             {
 340                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 341                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 342
 343                 /* They also don't support advanced CPUID functions. */
 344                 ExtendedCPUID = FALSE;
 345             }
 346
 347             /* Set the current features */
 348             CpuFeatures = Reg[3];
 349
 350             break;
 351
 352         /* Cyrix CPUs */
 353         case CPU_CYRIX:
 354
 355             /* FIXME: CMPXCGH8B */
 356
 357             break;
 358
 359         /* Transmeta CPUs */
 360         case CPU_TRANSMETA:
 361
 362             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 363             if ((Reg[0] & 0x0FFF) >= 0x0542)
 364             {
 365                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 366                 FeatureBits |= KF_CMPXCHG8B;
 367             }
 368
 369             break;
 370
 371         /* Centaur, IDT, Rise and VIA CPUs */
 372         case CPU_CENTAUR:
 373         case CPU_RISE:
 374
 375             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 376                However, this feature exists and operates properly without any additional steps. */
 377             FeatureBits |= KF_CMPXCHG8B;
 378
 379             break;
 380     }
 381
 382     /* Convert all CPUID Feature bits into our format */
 383     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 384     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 385     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 386     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 387     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 388     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 389     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 390     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 391     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 392     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 393     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 394     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 395     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 396     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 397
 398     /* Check if the CPU has hyper-threading */
 399     if (CpuFeatures & 0x10000000)
 400     {
 401         /* Set the number of logical CPUs */
 402         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 403         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 404         {
 405             /* We're on dual-core */
 406             KiSMTProcessorsPresent = TRUE;
 407         }
 408     }
 409     else
 410     {
 411         /* We only have a single CPU */
 412         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 413     }
 414
 415     /* Check if CPUID 0x80000000 is supported */
 416     if (ExtendedCPUID)
 417     {
 418         /* Do the call */
 419         CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
 420         if ((Reg[0] & 0xffffff00) == 0x80000000)
 421         {
 422             /* Check if CPUID 0x80000001 is supported */
 423             if (Reg[0] >= 0x80000001)
 424             {
 425                 /* Check which extended features are available. */
 426                 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
 427
 428                 /* Check if NX-bit is supported */
 429                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 430
 431                 /* Now handle each features for each CPU Vendor */
 432                 switch (Vendor)
 433                 {
 434                     case CPU_AMD:
 435                     case CPU_CENTAUR:
 436                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 437                         break;
 438                 }
 439             }
 440         }
 441     }
 442
 443     /* Return the Feature Bits */
 444     return FeatureBits;
 445 }
 446
 447 VOID
 448 NTAPI
 449 KiGetCacheInformation(VOID)
 450 {
 451     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 452     ULONG Vendor;
 453     ULONG Data[4], Dummy;
 454     ULONG CacheRequests = 0, i;
 455     ULONG CurrentRegister;
 456     UCHAR RegisterByte;
 457     ULONG Size, Associativity = 0, CacheLine = 64, CurrentSize = 0;
 458     BOOLEAN FirstPass = TRUE;
 459
 460     /* Set default L2 size */
 461     Pcr->SecondLevelCacheSize = 0;
 462
 463     /* Get the Vendor ID and make sure we support CPUID */
 464     Vendor = KiGetCpuVendor();
 465     if (!Vendor) return;
 466
 467     /* Check the Vendor ID */
 468     switch (Vendor)
 469     {
 470         /* Handle Intel case */
 471         case CPU_INTEL:
 472
 473             /*Check if we support CPUID 2 */
 474             CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
 475             if (Data[0] >= 2)
 476             {
 477                 /* We need to loop for the number of times CPUID will tell us to */
 478                 do
 479                 {
 480                     /* Do the CPUID call */
 481                     CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
 482
 483                     /* Check if it was the first call */
 484                     if (FirstPass)
 485                     {
 486                         /*
 487                          * The number of times to loop is the first byte. Read
 488                          * it and then destroy it so we don't get confused.
 489                          */
 490                         CacheRequests = Data[0] & 0xFF;
 491                         Data[0] &= 0xFFFFFF00;
 492
 493                         /* Don't go over this again */
 494                         FirstPass = FALSE;
 495                     }
 496
 497                     /* Loop all 4 registers */
 498                     for (i = 0; i < 4; i++)
 499                     {
 500                         /* Get the current register */
 501                         CurrentRegister = Data[i];
 502
 503                         /*
 504                          * If the upper bit is set, then this register should
 505                          * be skipped.
 506                          */
 507                         if (CurrentRegister & 0x80000000) continue;
 508
 509                         /* Keep looping for every byte inside this register */
 510                         while (CurrentRegister)
 511                         {
 512                             /* Read a byte, skip a byte. */
 513                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 514                             CurrentRegister >>= 8;
 515                             if (!RegisterByte) continue;
 516
 517                             /*
 518                              * Valid values are from 0x40 (0 bytes) to 0x49
 519                              * (32MB), or from 0x80 to 0x89 (same size but
 520                              * 8-way associative.
 521                              */
 522                             if (((RegisterByte > 0x40) && (RegisterByte <= 0x47)) ||
 523                                 ((RegisterByte > 0x78) && (RegisterByte <= 0x7C)) ||
 524                                 ((RegisterByte > 0x80) && (RegisterByte <= 0x85)))
 525                             {
 526                                 /* Compute associativity */
 527                                 Associativity = 4;
 528                                 if (RegisterByte >= 0x79) Associativity = 8;
 529
 530                                 /* Mask out only the first nibble */
 531                                 RegisterByte &= 0x07;
 532
 533                                 /* Check if this cache is bigger than the last */
 534                                 Size = 0x10000 << RegisterByte;
 535                                 if ((Size / Associativity) > CurrentSize)
 536                                 {
 537                                     /* Set the L2 Cache Size and Associativity */
 538                                     CurrentSize = Size / Associativity;
 539                                     Pcr->SecondLevelCacheSize = Size;
 540                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 541                                 }
 542                             }
 543                             else if ((RegisterByte > 0x21) && (RegisterByte <= 0x29))
 544                             {
 545                                 /* Set minimum cache line size */
 546                                 if (CacheLine < 128) CacheLine = 128;
 547
 548                                 /* Hard-code size/associativity */
 549                                 Associativity = 8;
 550                                 switch (RegisterByte)
 551                                 {
 552                                     case 0x22:
 553                                         Size = 512 * 1024;
 554                                         Associativity = 4;
 555                                         break;
 556
 557                                     case 0x23:
 558                                         Size = 1024 * 1024;
 559                                         break;
 560
 561                                     case 0x25:
 562                                         Size = 2048 * 1024;
 563                                         break;
 564
 565                                     case 0x29:
 566                                         Size = 4096 * 1024;
 567                                         break;
 568
 569                                     default:
 570                                         Size = 0;
 571                                         break;
 572                                 }
 573
 574                                 /* Check if this cache is bigger than the last */
 575                                 if ((Size / Associativity) > CurrentSize)
 576                                 {
 577                                     /* Set the L2 Cache Size and Associativity */
 578                                     CurrentSize = Size / Associativity;
 579                                     Pcr->SecondLevelCacheSize = Size;
 580                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 581                                 }
 582                             }
 583                             else if (((RegisterByte > 0x65) && (RegisterByte < 0x69)) ||
 584                                       (RegisterByte == 0x2C) || (RegisterByte == 0xF0))
 585                             {
 586                                 /* Indicates L1 cache line of 64 bytes */
 587                                 KePrefetchNTAGranularity = 64;
 588                             }
 589                             else if (RegisterByte == 0xF1)
 590                             {
 591                                 /* Indicates L1 cache line of 128 bytes */
 592                                 KePrefetchNTAGranularity = 128;
 593                             }
 594                             else if (((RegisterByte >= 0x4A) && (RegisterByte <= 0x4C)) ||
 595                                       (RegisterByte == 0x78) ||
 596                                       (RegisterByte == 0x7D) ||
 597                                       (RegisterByte == 0x7F) ||
 598                                       (RegisterByte == 0x86) ||
 599                                       (RegisterByte == 0x87))
 600                             {
 601                                 /* Set minimum cache line size */
 602                                 if (CacheLine < 64) CacheLine = 64;
 603
 604                                 /* Hard-code size/associativity */
 605                                 switch (RegisterByte)
 606                                 {
 607                                     case 0x4A:
 608                                         Size = 4 * 1024 * 1024;
 609                                         Associativity = 8;
 610                                         break;
 611
 612                                     case 0x4B:
 613                                         Size = 6 * 1024 * 1024;
 614                                         Associativity = 12;
 615                                         break;
 616
 617                                     case 0x4C:
 618                                         Size = 8 * 1024 * 1024;
 619                                         Associativity = 16;
 620                                         break;
 621
 622                                     case 0x78:
 623                                         Size = 1 * 1024 * 1024;
 624                                         Associativity = 4;
 625                                         break;
 626
 627                                     case 0x7D:
 628                                         Size = 2 * 1024 * 1024;
 629                                         Associativity = 8;
 630                                         break;
 631
 632                                     case 0x7F:
 633                                         Size = 512 * 1024;
 634                                         Associativity = 2;
 635                                         break;
 636
 637                                     case 0x86:
 638                                         Size = 512 * 1024;
 639                                         Associativity = 4;
 640                                         break;
 641
 642                                     case 0x87:
 643                                         Size = 1 * 1024 * 1024;
 644                                         Associativity = 8;
 645                                         break;
 646
 647                                     default:
 648                                         Size = 0;
 649                                         break;
 650                                 }
 651
 652                                 /* Check if this cache is bigger than the last */
 653                                 if ((Size / Associativity) > CurrentSize)
 654                                 {
 655                                     /* Set the L2 Cache Size and Associativity */
 656                                     CurrentSize = Size / Associativity;
 657                                     Pcr->SecondLevelCacheSize = Size;
 658                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 659                                 }
 660                             }
 661                         }
 662                     }
 663                 } while (--CacheRequests);
 664             }
 665             break;
 666
 667         case CPU_AMD:
 668
 669             /* Check if we support CPUID 0x80000005 */
 670             CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 671             if (Data[0] >= 0x80000006)
 672             {
 673                 /* Get L1 size first */
 674                 CPUID(0x80000005, &Data[0], &Data[1], &Data[2], &Data[3]);
 675                 KePrefetchNTAGranularity = Data[2] & 0xFF;
 676
 677                 /* Check if we support CPUID 0x80000006 */
 678                 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 679                 if (Data[0] >= 0x80000006)
 680                 {
 681                     /* Get 2nd level cache and tlb size */
 682                     CPUID(0x80000006, &Data[0], &Data[1], &Data[2], &Data[3]);
 683
 684                     /* Cache line size */
 685                     CacheLine = Data[2] & 0xFF;
 686
 687                     /* Hardcode associativity */
 688                     RegisterByte = Data[2] >> 12;
 689                     switch (RegisterByte)
 690                     {
 691                         case 2:
 692                             Associativity = 2;
 693                             break;
 694
 695                         case 4:
 696                             Associativity = 4;
 697                             break;
 698
 699                         case 6:
 700                             Associativity = 8;
 701                             break;
 702
 703                         case 8:
 704                         case 15:
 705                             Associativity = 16;
 706                             break;
 707
 708                         default:
 709                             Associativity = 1;
 710                             break;
 711                     }
 712
 713                     /* Compute size */
 714                     Size = (Data[2] >> 16) << 10;
 715
 716                     /* Hack for Model 6, Steping 300 */
 717                     if ((KeGetCurrentPrcb()->CpuType == 6) &&
 718                         (KeGetCurrentPrcb()->CpuStep == 0x300))
 719                     {
 720                         /* Stick 64K in there */
 721                         Size = 64 * 1024;
 722                     }
 723
 724                     /* Set the L2 Cache Size and associativity */
 725                     Pcr->SecondLevelCacheSize = Size;
 726                     Pcr->SecondLevelCacheAssociativity = Associativity;
 727                 }
 728             }
 729             break;
 730
 731         case CPU_CYRIX:
 732         case CPU_TRANSMETA:
 733         case CPU_CENTAUR:
 734         case CPU_RISE:
 735
 736             /* FIXME */
 737             break;
 738     }
 739
 740     /* Set the cache line */
 741     if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
 742     DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
 743             KePrefetchNTAGranularity,
 744             Pcr->SecondLevelCacheSize,
 745             KeLargestCacheLine,
 746             Pcr->SecondLevelCacheAssociativity);
 747 }
 748
 749 VOID
 750 NTAPI
 751 KiSetCR0Bits(VOID)
 752 {
 753     ULONG Cr0;
 754
 755     /* Save current CR0 */
 756     Cr0 = __readcr0();
 757
 758     /* If this is a 486, enable Write-Protection */
 759     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 760
 761     /* Set new Cr0 */
 762     __writecr0(Cr0);
 763 }
 764
 765 VOID
 766 NTAPI
 767 KiInitializeTSS2(IN PKTSS Tss,
 768                  IN PKGDTENTRY TssEntry OPTIONAL)
 769 {
 770     PUCHAR p;
 771
 772     /* Make sure the GDT Entry is valid */
 773     if (TssEntry)
 774     {
 775         /* Set the Limit */
 776         TssEntry->LimitLow = sizeof(KTSS) - 1;
 777         TssEntry->HighWord.Bits.LimitHi = 0;
 778     }
 779
 780     /* Now clear the I/O Map */
 781     ASSERT(IOPM_COUNT == 1);
 782     RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
 783
 784     /* Initialize Interrupt Direction Maps */
 785     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 786     RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
 787
 788     /* Add DPMI support for interrupts */
 789     p[0] = 4;
 790     p[3] = 0x18;
 791     p[4] = 0x18;
 792
 793     /* Initialize the default Interrupt Direction Map */
 794     p = Tss->IntDirectionMap;
 795     RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
 796
 797     /* Add DPMI support */
 798     p[0] = 4;
 799     p[3] = 0x18;
 800     p[4] = 0x18;
 801 }
 802
 803 VOID
 804 NTAPI
 805 KiInitializeTSS(IN PKTSS Tss)
 806 {
 807     /* Set an invalid map base */
 808     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 809
 810     /* Disable traps during Task Switches */
 811     Tss->Flags = 0;
 812
 813     /* Set LDT and Ring 0 SS */
 814     Tss->LDT = 0;
 815     Tss->Ss0 = KGDT_R0_DATA;
 816 }
 817
 818 VOID
 819 FASTCALL
 820 Ki386InitializeTss(IN PKTSS Tss,
 821                    IN PKIDTENTRY Idt,
 822                    IN PKGDTENTRY Gdt)
 823 {
 824     PKGDTENTRY TssEntry, TaskGateEntry;
 825
 826     /* Initialize the boot TSS. */
 827     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 828     TssEntry->HighWord.Bits.Type = I386_TSS;
 829     TssEntry->HighWord.Bits.Pres = 1;
 830     TssEntry->HighWord.Bits.Dpl = 0;
 831     KiInitializeTSS2(Tss, TssEntry);
 832     KiInitializeTSS(Tss);
 833
 834     /* Load the task register */
 835     Ke386SetTr(KGDT_TSS);
 836
 837     /* Setup the Task Gate for Double Fault Traps */
 838     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 839     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 840     TaskGateEntry->HighWord.Bits.Pres = 1;
 841     TaskGateEntry->HighWord.Bits.Dpl = 0;
 842     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 843
 844     /* Initialize the TSS used for handling double faults. */
 845     Tss = (PKTSS)KiDoubleFaultTSS;
 846     KiInitializeTSS(Tss);
 847     Tss->CR3 = __readcr3();
 848     Tss->Esp0 = KiDoubleFaultStack;
 849     Tss->Esp = KiDoubleFaultStack;
 850     Tss->Eip = PtrToUlong(KiTrap08);
 851     Tss->Cs = KGDT_R0_CODE;
 852     Tss->Fs = KGDT_R0_PCR;
 853     Tss->Ss = Ke386GetSs();
 854     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 855     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 856
 857     /* Setup the Double Trap TSS entry in the GDT */
 858     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 859     TssEntry->HighWord.Bits.Type = I386_TSS;
 860     TssEntry->HighWord.Bits.Pres = 1;
 861     TssEntry->HighWord.Bits.Dpl = 0;
 862     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 863     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 864     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 865     TssEntry->LimitLow = KTSS_IO_MAPS;
 866
 867     /* Now setup the NMI Task Gate */
 868     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 869     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 870     TaskGateEntry->HighWord.Bits.Pres = 1;
 871     TaskGateEntry->HighWord.Bits.Dpl = 0;
 872     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 873
 874     /* Initialize the actual TSS */
 875     Tss = (PKTSS)KiNMITSS;
 876     KiInitializeTSS(Tss);
 877     Tss->CR3 = __readcr3();
 878     Tss->Esp0 = KiDoubleFaultStack;
 879     Tss->Esp = KiDoubleFaultStack;
 880     Tss->Eip = PtrToUlong(KiTrap02);
 881     Tss->Cs = KGDT_R0_CODE;
 882     Tss->Fs = KGDT_R0_PCR;
 883     Tss->Ss = Ke386GetSs();
 884     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 885     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 886
 887     /* And its associated TSS Entry */
 888     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 889     TssEntry->HighWord.Bits.Type = I386_TSS;
 890     TssEntry->HighWord.Bits.Pres = 1;
 891     TssEntry->HighWord.Bits.Dpl = 0;
 892     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 893     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 894     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 895     TssEntry->LimitLow = KTSS_IO_MAPS;
 896 }
 897
 898 VOID
 899 NTAPI
 900 KeFlushCurrentTb(VOID)
 901 {
 902     /* Flush the TLB by resetting CR3 */
 903     __writecr3(__readcr3());
 904 }
 905
 906 VOID
 907 NTAPI
 908 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 909 {
 910     PKGDTENTRY TssEntry;
 911
 912     //
 913     // Restore the CR registers
 914     //
 915     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 916     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 917     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 918     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 919
 920     //
 921     // Restore the DR registers
 922     //
 923     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 924     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 925     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 926     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 927     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 928     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 929
 930     //
 931     // Restore GDT and IDT
 932     //
 933     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 934     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 935
 936     //
 937     // Clear the busy flag so we don't crash if we reload the same selector
 938     //
 939     TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
 940                             ProcessorState->SpecialRegisters.Tr);
 941     TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
 942
 943     //
 944     // Restore TSS and LDT
 945     //
 946     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 947     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 948 }
 949
 950 VOID
 951 NTAPI
 952 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 953 {
 954     /* Save the CR registers */
 955     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 956     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 957     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 958     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 959                                            __readcr4() : 0;
 960
 961     /* Save the DR registers */
 962     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 963     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 964     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 965     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 966     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 967     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 968     __writedr(7, 0);
 969
 970     /* Save GDT, IDT, LDT and TSS */
 971     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 972     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 973     ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
 974     ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
 975 }
 976
 977 VOID
 978 NTAPI
 979 KiInitializeMachineType(VOID)
 980 {
 981     /* Set the Machine Type we got from NTLDR */
 982     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 983 }
 984
 985 ULONG_PTR
 986 NTAPI
 987 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 988 {
 989     /* Set CS and ESP */
 990     WRMSR(0x174, KGDT_R0_CODE);
 991     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
 992
 993     /* Set LSTAR */
 994     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
 995     return 0;
 996 }
 997
 998 VOID
 999 NTAPI
1000 KiRestoreFastSyscallReturnState(VOID)
1001 {
1002     /* Check if the CPU Supports fast system call */
1003     if (KeFeatureBits & KF_FAST_SYSCALL)
1004     {
1005         /* Check if it has been disabled */
1006         if (!KiFastSystemCallDisable)
1007         {
1008             /* Do an IPI to enable it */
1009             KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
1010
1011             /* It's enabled, so use the proper exit stub */
1012             KiFastCallExitHandler = KiSystemCallSysExitReturn;
1013             DPRINT1("Support for SYSENTER detected.\n");
1014         }
1015         else
1016         {
1017             /* Disable fast system call */
1018             KeFeatureBits &= ~KF_FAST_SYSCALL;
1019             KiFastCallExitHandler = KiSystemCallTrapReturn;
1020             DPRINT1("Support for SYSENTER disabled.\n");
1021         }
1022     }
1023     else
1024     {
1025         /* Use the IRET handler */
1026         KiFastCallExitHandler = KiSystemCallTrapReturn;
1027         DPRINT1("No support for SYSENTER detected.\n");
1028     }
1029 }
1030
1031 ULONG_PTR
1032 NTAPI
1033 Ki386EnableDE(IN ULONG_PTR Context)
1034 {
1035     /* Enable DE */
1036     __writecr4(__readcr4() | CR4_DE);
1037     return 0;
1038 }
1039
1040 ULONG_PTR
1041 NTAPI
1042 Ki386EnableFxsr(IN ULONG_PTR Context)
1043 {
1044     /* Enable FXSR */
1045     __writecr4(__readcr4() | CR4_FXSR);
1046     return 0;
1047 }
1048
1049 ULONG_PTR
1050 NTAPI
1051 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
1052 {
1053     PKIDTENTRY IdtEntry;
1054
1055     /* Get the IDT Entry for Interrupt 0x13 */
1056     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
1057
1058     /* Set it up */
1059     IdtEntry->Selector = KGDT_R0_CODE;
1060     IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
1061     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
1062     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
1063     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
1064     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
1065
1066     /* Enable XMMI exceptions */
1067     __writecr4(__readcr4() | CR4_XMMEXCPT);
1068     return 0;
1069 }
1070
1071 VOID
1072 NTAPI
1073 KiI386PentiumLockErrataFixup(VOID)
1074 {
1075     KDESCRIPTOR IdtDescriptor;
1076     PKIDTENTRY NewIdt, NewIdt2;
1077
1078     /* Allocate memory for a new IDT */
1079     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
1080
1081     /* Put everything after the first 7 entries on a new page */
1082     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
1083
1084     /* Disable interrupts */
1085     _disable();
1086
1087     /* Get the current IDT and copy it */
1088     __sidt(&IdtDescriptor.Limit);
1089     RtlCopyMemory(NewIdt2,
1090                   (PVOID)IdtDescriptor.Base,
1091                   IdtDescriptor.Limit + 1);
1092     IdtDescriptor.Base = (ULONG)NewIdt2;
1093
1094     /* Set the new IDT */
1095     __lidt(&IdtDescriptor.Limit);
1096     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
1097
1098     /* Restore interrupts */
1099     _enable();
1100
1101     /* Set the first 7 entries as read-only to produce a fault */
1102     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
1103 }
1104
1105 BOOLEAN
1106 NTAPI
1107 KeDisableInterrupts(VOID)
1108 {
1109     ULONG Flags;
1110     BOOLEAN Return;
1111
1112     /* Get EFLAGS and check if the interrupt bit is set */
1113     Flags = __readeflags();
1114     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
1115
1116     /* Disable interrupts */
1117     _disable();
1118     return Return;
1119 }
1120
1121 BOOLEAN
1122 NTAPI
1123 KeInvalidateAllCaches(VOID)
1124 {
1125     /* Only supported on Pentium Pro and higher */
1126     if (KeI386CpuType < 6) return FALSE;
1127
1128     /* Invalidate all caches */
1129     __wbinvd();
1130     return TRUE;
1131 }
1132
1133 VOID
1134 FASTCALL
1135 KeZeroPages(IN PVOID Address,
1136             IN ULONG Size)
1137 {
1138     /* Not using XMMI in this routine */
1139     RtlZeroMemory(Address, Size);
1140 }
1141
1142 VOID
1143 NTAPI
1144 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
1145                      IN PKEXCEPTION_FRAME ExceptionFrame)
1146 {
1147     PKPRCB Prcb = KeGetCurrentPrcb();
1148
1149     //
1150     // Save full context
1151     //
1152     Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
1153                                                      CONTEXT_DEBUG_REGISTERS;
1154     KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
1155
1156     //
1157     // Save control registers
1158     //
1159     KiSaveProcessorControlState(&Prcb->ProcessorState);
1160 }
1161
1162 BOOLEAN
1163 NTAPI
1164 KiIsNpxPresent(VOID)
1165 {
1166     ULONG Cr0;
1167     USHORT Magic;
1168
1169     /* Set magic */
1170     Magic = 0xFFFF;
1171
1172     /* Read CR0 and mask out FPU flags */
1173     Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
1174
1175     /* Store on FPU stack */
1176     asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
1177
1178     /* Magic should now be cleared */
1179     if (Magic & 0xFF)
1180     {
1181         /* You don't have an FPU -- enable emulation for now */
1182         __writecr0(Cr0 | CR0_EM | CR0_TS);
1183         return FALSE;
1184     }
1185
1186     /* You have an FPU, enable it */
1187     Cr0 |= CR0_ET;
1188
1189     /* Enable INT 16 on 486 and higher */
1190     if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
1191
1192     /* Set FPU state */
1193     __writecr0(Cr0 | CR0_EM | CR0_TS);
1194     return TRUE;
1195 }
1196
1197 BOOLEAN
1198 NTAPI
1199 KiIsNpxErrataPresent(VOID)
1200 {
1201     BOOLEAN ErrataPresent;
1202     ULONG Cr0;
1203     volatile double Value1, Value2;
1204
1205     /* Disable interrupts */
1206     _disable();
1207
1208     /* Read CR0 and remove FPU flags */
1209     Cr0 = __readcr0();
1210     __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1211
1212     /* Initialize FPU state */
1213     asm volatile ("fninit");
1214
1215     /* Multiply the magic values and divide, we should get the result back */
1216     Value1 = 4195835.0;
1217     Value2 = 3145727.0;
1218     ErrataPresent = (Value1 * Value2 / 3145727.0) != 4195835.0;
1219
1220     /* Restore CR0 */
1221     __writecr0(Cr0);
1222
1223     /* Enable interrupts */
1224     _enable();
1225
1226     /* Return if there's an errata */
1227     return ErrataPresent;
1228 }
1229
1230 NTAPI
1231 VOID
1232 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
1233 {
1234     ULONG EFlags, Cr0;
1235     PKTHREAD Thread, NpxThread;
1236     PFX_SAVE_AREA FxSaveArea;
1237
1238     /* Save volatiles and disable interrupts */
1239     EFlags = __readeflags();
1240     _disable();
1241
1242     /* Save the PCR and get the current thread */
1243     Thread = KeGetCurrentThread();
1244
1245     /* Check if we're already loaded */
1246     if (Thread->NpxState != NPX_STATE_LOADED)
1247     {
1248         /* If there's nothing to load, quit */
1249         if (!SaveArea) return;
1250
1251         /* Need FXSR support for this */
1252         ASSERT(KeI386FxsrPresent == TRUE);
1253
1254         /* Check for sane CR0 */
1255         Cr0 = __readcr0();
1256         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1257         {
1258             /* Mask out FPU flags */
1259             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1260         }
1261
1262         /* Get the NPX thread and check its FPU state */
1263         NpxThread = KeGetCurrentPrcb()->NpxThread;
1264         if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
1265         {
1266             /* Get the FX frame and store the state there */
1267             FxSaveArea = KiGetThreadNpxArea(NpxThread);
1268             Ke386FxSave(FxSaveArea);
1269
1270             /* NPX thread has lost its state */
1271             NpxThread->NpxState = NPX_STATE_NOT_LOADED;
1272         }
1273
1274         /* Now load NPX state from the NPX area */
1275         FxSaveArea = KiGetThreadNpxArea(Thread);
1276         Ke386FxStore(FxSaveArea);
1277     }
1278     else
1279     {
1280         /* Check for sane CR0 */
1281         Cr0 = __readcr0();
1282         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1283         {
1284             /* Mask out FPU flags */
1285             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1286         }
1287
1288         /* Get FX frame */
1289         FxSaveArea = KiGetThreadNpxArea(Thread);
1290         Thread->NpxState = NPX_STATE_NOT_LOADED;
1291
1292         /* Save state if supported by CPU */
1293         if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
1294     }
1295
1296     /* Now save the FN state wherever it was requested */
1297     if (SaveArea) Ke386FnSave(SaveArea);
1298
1299     /* Clear NPX thread */
1300     KeGetCurrentPrcb()->NpxThread = NULL;
1301
1302     /* Add the CR0 from the NPX frame */
1303     Cr0 |= NPX_STATE_NOT_LOADED;
1304     Cr0 |= FxSaveArea->Cr0NpxState;
1305     __writecr0(Cr0);
1306
1307     /* Restore interrupt state */
1308     __writeeflags(EFlags);
1309 }
1310
1311 /* PUBLIC FUNCTIONS **********************************************************/
1312
1313 /*
1314  * @implemented
1315  */
1316 VOID
1317 NTAPI
1318 KiCoprocessorError(VOID)
1319 {
1320     PFX_SAVE_AREA NpxArea;
1321
1322     /* Get the FPU area */
1323     NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
1324
1325     /* Set CR0_TS */
1326     NpxArea->Cr0NpxState = CR0_TS;
1327     __writecr0(__readcr0() | CR0_TS);
1328 }
1329
1330 /*
1331  * @implemented
1332  */
1333 NTSTATUS
1334 NTAPI
1335 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
1336 {
1337     PFNSAVE_FORMAT FpState;
1338     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
1339     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1340
1341     /* check if we are doing software emulation */
1342     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
1343
1344     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1345     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1346
1347     *((PVOID *) Save) = FpState;
1348 #ifdef __GNUC__
1349     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1350 #else
1351     __asm
1352     {
1353         fnsave [FpState]
1354     };
1355 #endif
1356
1357     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1358     return STATUS_SUCCESS;
1359 }
1360
1361 /*
1362  * @implemented
1363  */
1364 NTSTATUS
1365 NTAPI
1366 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1367 {
1368     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1369     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1370     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1371
1372 #ifdef __GNUC__
1373     asm volatile("fnclex\n\t");
1374     asm volatile("frstor %0\n\t" : "=m" (*FpState));
1375 #else
1376     __asm
1377     {
1378         fnclex
1379         frstor [FpState]
1380     };
1381 #endif
1382
1383     ExFreePool(FpState);
1384     return STATUS_SUCCESS;
1385 }
1386
1387 /*
1388  * @implemented
1389  */
1390 ULONG
1391 NTAPI
1392 KeGetRecommendedSharedDataAlignment(VOID)
1393 {
1394     /* Return the global variable */
1395     return KeLargestCacheLine;
1396 }
1397
1398 VOID
1399 NTAPI
1400 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1401                       IN PVOID Ignored1,
1402                       IN PVOID Ignored2,
1403                       IN PVOID Ignored3)
1404 {
1405     /* Signal this packet as done */
1406     KiIpiSignalPacketDone(PacketContext);
1407
1408     /* Flush the TB for the Current CPU */
1409     KeFlushCurrentTb();
1410 }
1411
1412 /*
1413  * @implemented
1414  */
1415 VOID
1416 NTAPI
1417 KeFlushEntireTb(IN BOOLEAN Invalid,
1418                 IN BOOLEAN AllProcessors)
1419 {
1420     KIRQL OldIrql;
1421 #ifdef CONFIG_SMP
1422     KAFFINITY TargetAffinity;
1423     PKPRCB Prcb = KeGetCurrentPrcb();
1424 #endif
1425
1426     /* Raise the IRQL for the TB Flush */
1427     OldIrql = KeRaiseIrqlToSynchLevel();
1428
1429 #ifdef CONFIG_SMP
1430     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1431
1432     /* Get the current processor affinity, and exclude ourselves */
1433     TargetAffinity = KeActiveProcessors;
1434     TargetAffinity &= ~Prcb->SetMember;
1435
1436     /* Make sure this is MP */
1437     if (TargetAffinity)
1438     {
1439         /* Send an IPI TB flush to the other processors */
1440         KiIpiSendPacket(TargetAffinity,
1441                         KiFlushTargetEntireTb,
1442                         NULL,
1443                         0,
1444                         NULL);
1445     }
1446 #endif
1447
1448     /* Flush the TB for the Current CPU, and update the flush stamp */
1449     KeFlushCurrentTb();
1450
1451 #ifdef CONFIG_SMP
1452     /* If this is MP, wait for the other processors to finish */
1453     if (TargetAffinity)
1454     {
1455         /* Sanity check */
1456         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1457
1458         /* FIXME: TODO */
1459         ASSERTMSG("Not yet implemented\n", FALSE);
1460     }
1461 #endif
1462
1463     /* Update the flush stamp and return to original IRQL */
1464     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1465     KeLowerIrql(OldIrql);
1466 }
1467
1468 /*
1469  * @implemented
1470  */
1471 VOID
1472 NTAPI
1473 KeSetDmaIoCoherency(IN ULONG Coherency)
1474 {
1475     /* Save the coherency globally */
1476     KiDmaIoCoherency = Coherency;
1477 }
1478
1479 /*
1480  * @implemented
1481  */
1482 KAFFINITY
1483 NTAPI
1484 KeQueryActiveProcessors(VOID)
1485 {
1486     PAGED_CODE();
1487
1488     /* Simply return the number of active processors */
1489     return KeActiveProcessors;
1490 }
1491
1492 /*
1493  * @implemented
1494  */
1495 VOID
1496 __cdecl
1497 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1498 {
1499     /* Capture the context */
1500     RtlCaptureContext(&State->ContextFrame);
1501
1502     /* Capture the control state */
1503     KiSaveProcessorControlState(State);
1504 }