reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The TSS to use for Double Fault Traps (INT 0x9) */
  18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  19
  20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  21 UCHAR KiNMITSS[KTSS_IO_MAPS];
  22
  23 /* CPU Features and Flags */
  24 ULONG KeI386CpuType;
  25 ULONG KeI386CpuStep;
  26 ULONG KeProcessorArchitecture;
  27 ULONG KeProcessorLevel;
  28 ULONG KeProcessorRevision;
  29 ULONG KeFeatureBits;
  30 ULONG KiFastSystemCallDisable;
  31 ULONG KeI386NpxPresent = 0;
  32 ULONG KiMXCsrMask = 0;
  33 ULONG MxcsrFeatureMask = 0;
  34 ULONG KeI386XMMIPresent = 0;
  35 ULONG KeI386FxsrPresent = 0;
  36 ULONG KeI386MachineType;
  37 ULONG Ke386Pae = FALSE;
  38 ULONG Ke386NoExecute = FALSE;
  39 ULONG KeLargestCacheLine = 0x40;
  40 ULONG KeDcacheFlushCount = 0;
  41 ULONG KeIcacheFlushCount = 0;
  42 ULONG KiDmaIoCoherency = 0;
  43 ULONG KePrefetchNTAGranularity = 32;
  44 CHAR KeNumberProcessors;
  45 KAFFINITY KeActiveProcessors = 1;
  46 BOOLEAN KiI386PentiumLockErrataPresent;
  47 BOOLEAN KiSMTProcessorsPresent;
  48
  49 /* The distance between SYSEXIT and IRETD return modes */
  50 UCHAR KiSystemCallExitAdjust;
  51
  52 /* The offset that was applied -- either 0 or the value above */
  53 UCHAR KiSystemCallExitAdjusted;
  54
  55 /* Whether the adjustment was already done once */
  56 BOOLEAN KiFastCallCopyDoneOnce;
  57
  58 /* Flush data */
  59 volatile LONG KiTbFlushTimeStamp;
  60
  61 /* CPU Signatures */
  62 static const CHAR CmpIntelID[]       = "GenuineIntel";
  63 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  64 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  65 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  66 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  67 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  68
  69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  70
  71 VOID
  72 NTAPI
  73 CPUID(IN ULONG InfoType,
  74       OUT PULONG CpuInfoEax,
  75       OUT PULONG CpuInfoEbx,
  76       OUT PULONG CpuInfoEcx,
  77       OUT PULONG CpuInfoEdx)
  78 {
  79     ULONG CpuInfo[4];
  80
  81     /* Perform the CPUID Operation */
  82     __cpuid((int*)CpuInfo, InfoType);
  83
  84     /* Return the results */
  85     *CpuInfoEax = CpuInfo[0];
  86     *CpuInfoEbx = CpuInfo[1];
  87     *CpuInfoEcx = CpuInfo[2];
  88     *CpuInfoEdx = CpuInfo[3];
  89 }
  90
  91 VOID
  92 NTAPI
  93 WRMSR(IN ULONG Register,
  94       IN LONGLONG Value)
  95 {
  96    /* Write to the MSR */
  97     __writemsr(Register, Value);
  98 }
  99
 100 LONGLONG
 101 FASTCALL
 102 RDMSR(IN ULONG Register)
 103 {
 104     /* Read from the MSR */
 105     return __readmsr(Register);
 106 }
 107
 108 /* FUNCTIONS *****************************************************************/
 109
 110 VOID
 111 NTAPI
 112 KiSetProcessorType(VOID)
 113 {
 114     ULONG EFlags, NewEFlags;
 115     ULONG Reg, Dummy;
 116     ULONG Stepping, Type;
 117
 118     /* Start by assuming no CPUID data */
 119     KeGetCurrentPrcb()->CpuID = 0;
 120
 121     /* Save EFlags */
 122     EFlags = __readeflags();
 123
 124     /* XOR out the ID bit and update EFlags */
 125     NewEFlags = EFlags ^ EFLAGS_ID;
 126     __writeeflags(NewEFlags);
 127
 128     /* Get them back and see if they were modified */
 129     NewEFlags = __readeflags();
 130     if (NewEFlags != EFlags)
 131     {
 132         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 133         EFlags |= EFLAGS_ID;
 134         __writeeflags(EFlags);
 135
 136         /* Peform CPUID 0 to see if CPUID 1 is supported */
 137         CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
 138         if (Reg > 0)
 139         {
 140             /* Do CPUID 1 now */
 141             CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
 142
 143             /*
 144              * Get the Stepping and Type. The stepping contains both the
 145              * Model and the Step, while the Type contains the returned Type.
 146              * We ignore the family.
 147              *
 148              * For the stepping, we convert this: zzzzzzxy into this: x0y
 149              */
 150             Stepping = Reg & 0xF0;
 151             Stepping <<= 4;
 152             Stepping += (Reg & 0xFF);
 153             Stepping &= 0xF0F;
 154             Type = Reg & 0xF00;
 155             Type >>= 8;
 156
 157             /* Save them in the PRCB */
 158             KeGetCurrentPrcb()->CpuID = TRUE;
 159             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 160             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 161         }
 162         else
 163         {
 164             DPRINT1("CPUID Support lacking\n");
 165         }
 166     }
 167     else
 168     {
 169         DPRINT1("CPUID Support lacking\n");
 170     }
 171
 172     /* Restore EFLAGS */
 173     __writeeflags(EFlags);
 174 }
 175
 176 ULONG
 177 NTAPI
 178 KiGetCpuVendor(VOID)
 179 {
 180     PKPRCB Prcb = KeGetCurrentPrcb();
 181     ULONG Vendor[5];
 182     ULONG Temp;
 183
 184     /* Assume no Vendor ID and fail if no CPUID Support. */
 185     Prcb->VendorString[0] = 0;
 186     if (!Prcb->CpuID) return 0;
 187
 188     /* Get the Vendor ID and null-terminate it */
 189     CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
 190     Vendor[4] = 0;
 191
 192     /* Re-arrange vendor string */
 193     Temp = Vendor[2];
 194     Vendor[2] = Vendor[3];
 195     Vendor[3] = Temp;
 196
 197     /* Copy it to the PRCB and null-terminate it again */
 198     RtlCopyMemory(Prcb->VendorString,
 199                   &Vendor[1],
 200                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 201     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 202
 203     /* Now check the CPU Type */
 204     if (!strcmp((PCHAR)Prcb->VendorString, CmpIntelID))
 205     {
 206         return CPU_INTEL;
 207     }
 208     else if (!strcmp((PCHAR)Prcb->VendorString, CmpAmdID))
 209     {
 210         return CPU_AMD;
 211     }
 212     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCyrixID))
 213     {
 214         DPRINT1("Cyrix CPU support not fully tested!\n");
 215         return CPU_CYRIX;
 216     }
 217     else if (!strcmp((PCHAR)Prcb->VendorString, CmpTransmetaID))
 218     {
 219         DPRINT1("Transmeta CPU support not fully tested!\n");
 220         return CPU_TRANSMETA;
 221     }
 222     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCentaurID))
 223     {
 224         DPRINT1("Centaur CPU support not fully tested!\n");
 225         return CPU_CENTAUR;
 226     }
 227     else if (!strcmp((PCHAR)Prcb->VendorString, CmpRiseID))
 228     {
 229         DPRINT1("Rise CPU support not fully tested!\n");
 230         return CPU_RISE;
 231     }
 232
 233     /* Invalid CPU */
 234     return 0;
 235 }
 236
 237 ULONG
 238 NTAPI
 239 KiGetFeatureBits(VOID)
 240 {
 241     PKPRCB Prcb = KeGetCurrentPrcb();
 242     ULONG Vendor;
 243     ULONG FeatureBits = KF_WORKING_PTE;
 244     ULONG Reg[4], Dummy;
 245     BOOLEAN ExtendedCPUID = TRUE;
 246     ULONG CpuFeatures = 0;
 247
 248     /* Get the Vendor ID */
 249     Vendor = KiGetCpuVendor();
 250
 251     /* Make sure we got a valid vendor ID at least. */
 252     if (!Vendor) return FeatureBits;
 253
 254     /* Get the CPUID Info. Features are in Reg[3]. */
 255     CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
 256
 257     /* Set the initial APIC ID */
 258     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 259
 260     switch (Vendor)
 261     {
 262         /* Intel CPUs */
 263         case CPU_INTEL:
 264
 265             /* Check if it's a P6 */
 266             if (Prcb->CpuType == 6)
 267             {
 268                 /* Perform the special sequence to get the MicroCode Signature */
 269                 WRMSR(0x8B, 0);
 270                 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
 271                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 272             }
 273             else if (Prcb->CpuType == 5)
 274             {
 275                 /* On P5, enable workaround for the LOCK errata. */
 276                 KiI386PentiumLockErrataPresent = TRUE;
 277             }
 278
 279             /* Check for broken P6 with bad SMP PTE implementation */
 280             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 281                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 282             {
 283                 /* Remove support for correct PTE support. */
 284                 FeatureBits &= ~KF_WORKING_PTE;
 285             }
 286
 287             /* Check if the CPU is too old to support SYSENTER */
 288             if ((Prcb->CpuType < 6) ||
 289                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 290             {
 291                 /* Disable it */
 292                 Reg[3] &= ~0x800;
 293             }
 294
 295             /* Set the current features */
 296             CpuFeatures = Reg[3];
 297
 298             break;
 299
 300         /* AMD CPUs */
 301         case CPU_AMD:
 302
 303             /* Check if this is a K5 or K6. (family 5) */
 304             if ((Reg[0] & 0x0F00) == 0x0500)
 305             {
 306                 /* Get the Model Number */
 307                 switch (Reg[0] & 0x00F0)
 308                 {
 309                     /* Model 1: K5 - 5k86 (initial models) */
 310                     case 0x0010:
 311
 312                         /* Check if this is Step 0 or 1. They don't support PGE */
 313                         if ((Reg[0] & 0x000F) > 0x03) break;
 314
 315                     /* Model 0: K5 - SSA5 */
 316                     case 0x0000:
 317
 318                         /* Model 0 doesn't support PGE at all. */
 319                         Reg[3] &= ~0x2000;
 320                         break;
 321
 322                     /* Model 8: K6-2 */
 323                     case 0x0080:
 324
 325                         /* K6-2, Step 8 and over have support for MTRR. */
 326                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 327                         break;
 328
 329                     /* Model 9: K6-III
 330                        Model D: K6-2+, K6-III+ */
 331                     case 0x0090:
 332                     case 0x00D0:
 333
 334                         FeatureBits |= KF_AMDK6MTRR;
 335                         break;
 336                 }
 337             }
 338             else if((Reg[0] & 0x0F00) < 0x0500)
 339             {
 340                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 341                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 342
 343                 /* They also don't support advanced CPUID functions. */
 344                 ExtendedCPUID = FALSE;
 345             }
 346
 347             /* Set the current features */
 348             CpuFeatures = Reg[3];
 349
 350             break;
 351
 352         /* Cyrix CPUs */
 353         case CPU_CYRIX:
 354
 355             /* FIXME: CMPXCGH8B */
 356
 357             break;
 358
 359         /* Transmeta CPUs */
 360         case CPU_TRANSMETA:
 361
 362             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 363             if ((Reg[0] & 0x0FFF) >= 0x0542)
 364             {
 365                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 366                 FeatureBits |= KF_CMPXCHG8B;
 367             }
 368
 369             break;
 370
 371         /* Centaur, IDT, Rise and VIA CPUs */
 372         case CPU_CENTAUR:
 373         case CPU_RISE:
 374
 375             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 376                However, this feature exists and operates properly without any additional steps. */
 377             FeatureBits |= KF_CMPXCHG8B;
 378
 379             break;
 380     }
 381
 382     /* Convert all CPUID Feature bits into our format */
 383     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 384     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 385     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 386     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 387     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 388     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 389     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 390     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 391     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 392     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 393     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 394     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 395     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 396     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 397
 398     /* Check if the CPU has hyper-threading */
 399     if (CpuFeatures & 0x10000000)
 400     {
 401         /* Set the number of logical CPUs */
 402         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 403         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 404         {
 405             /* We're on dual-core */
 406             KiSMTProcessorsPresent = TRUE;
 407         }
 408     }
 409     else
 410     {
 411         /* We only have a single CPU */
 412         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 413     }
 414
 415     /* Check if CPUID 0x80000000 is supported */
 416     if (ExtendedCPUID)
 417     {
 418         /* Do the call */
 419         CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
 420         if ((Reg[0] & 0xffffff00) == 0x80000000)
 421         {
 422             /* Check if CPUID 0x80000001 is supported */
 423             if (Reg[0] >= 0x80000001)
 424             {
 425                 /* Check which extended features are available. */
 426                 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
 427
 428                 /* Check if NX-bit is supported */
 429                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 430
 431                 /* Now handle each features for each CPU Vendor */
 432                 switch (Vendor)
 433                 {
 434                     case CPU_AMD:
 435                     case CPU_CENTAUR:
 436                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 437                         break;
 438                 }
 439             }
 440         }
 441     }
 442
 443     /* Return the Feature Bits */
 444     return FeatureBits;
 445 }
 446
 447 VOID
 448 NTAPI
 449 KiGetCacheInformation(VOID)
 450 {
 451     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 452     ULONG Vendor;
 453     ULONG Data[4], Dummy;
 454     ULONG CacheRequests = 0, i;
 455     ULONG CurrentRegister;
 456     UCHAR RegisterByte;
 457     ULONG Size, Associativity = 0, CacheLine = 64, CurrentSize = 0;
 458     BOOLEAN FirstPass = TRUE;
 459
 460     /* Set default L2 size */
 461     Pcr->SecondLevelCacheSize = 0;
 462
 463     /* Get the Vendor ID and make sure we support CPUID */
 464     Vendor = KiGetCpuVendor();
 465     if (!Vendor) return;
 466
 467     /* Check the Vendor ID */
 468     switch (Vendor)
 469     {
 470         /* Handle Intel case */
 471         case CPU_INTEL:
 472
 473             /*Check if we support CPUID 2 */
 474             CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
 475             if (Data[0] >= 2)
 476             {
 477                 /* We need to loop for the number of times CPUID will tell us to */
 478                 do
 479                 {
 480                     /* Do the CPUID call */
 481                     CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
 482
 483                     /* Check if it was the first call */
 484                     if (FirstPass)
 485                     {
 486                         /*
 487                          * The number of times to loop is the first byte. Read
 488                          * it and then destroy it so we don't get confused.
 489                          */
 490                         CacheRequests = Data[0] & 0xFF;
 491                         Data[0] &= 0xFFFFFF00;
 492
 493                         /* Don't go over this again */
 494                         FirstPass = FALSE;
 495                     }
 496
 497                     /* Loop all 4 registers */
 498                     for (i = 0; i < 4; i++)
 499                     {
 500                         /* Get the current register */
 501                         CurrentRegister = Data[i];
 502
 503                         /*
 504                          * If the upper bit is set, then this register should
 505                          * be skipped.
 506                          */
 507                         if (CurrentRegister & 0x80000000) continue;
 508
 509                         /* Keep looping for every byte inside this register */
 510                         while (CurrentRegister)
 511                         {
 512                             /* Read a byte, skip a byte. */
 513                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 514                             CurrentRegister >>= 8;
 515                             if (!RegisterByte) continue;
 516
 517                             /*
 518                              * Valid values are from 0x40 (0 bytes) to 0x49
 519                              * (32MB), or from 0x80 to 0x89 (same size but
 520                              * 8-way associative.
 521                              */
 522                             if (((RegisterByte > 0x40) && (RegisterByte <= 0x47)) ||
 523                                 ((RegisterByte > 0x78) && (RegisterByte <= 0x7C)) ||
 524                                 ((RegisterByte > 0x80) && (RegisterByte <= 0x85)))
 525                             {
 526                                 /* Compute associativity */
 527                                 Associativity = 4;
 528                                 if (RegisterByte >= 0x79) Associativity = 8;
 529
 530                                 /* Mask out only the first nibble */
 531                                 RegisterByte &= 0x07;
 532
 533                                 /* Check if this cache is bigger than the last */
 534                                 Size = 0x10000 << RegisterByte;
 535                                 if ((Size / Associativity) > CurrentSize)
 536                                 {
 537                                     /* Set the L2 Cache Size and Associativity */
 538                                     CurrentSize = Size / Associativity;
 539                                     Pcr->SecondLevelCacheSize = Size;
 540                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 541                                 }
 542                             }
 543                             else if ((RegisterByte > 0x21) && (RegisterByte <= 0x29))
 544                             {
 545                                 /* Set minimum cache line size */
 546                                 if (CacheLine < 128) CacheLine = 128;
 547
 548                                 /* Hard-code size/associativity */
 549                                 Associativity = 8;
 550                                 switch (RegisterByte)
 551                                 {
 552                                     case 0x22:
 553                                         Size = 512 * 1024;
 554                                         Associativity = 4;
 555                                         break;
 556
 557                                     case 0x23:
 558                                         Size = 1024 * 1024;
 559                                         break;
 560
 561                                     case 0x25:
 562                                         Size = 2048 * 1024;
 563                                         break;
 564
 565                                     case 0x29:
 566                                         Size = 4096 * 1024;
 567                                         break;
 568
 569                                     default:
 570                                         Size = 0;
 571                                         break;
 572                                 }
 573
 574                                 /* Check if this cache is bigger than the last */
 575                                 if ((Size / Associativity) > CurrentSize)
 576                                 {
 577                                     /* Set the L2 Cache Size and Associativity */
 578                                     CurrentSize = Size / Associativity;
 579                                     Pcr->SecondLevelCacheSize = Size;
 580                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 581                                 }
 582                             }
 583                             else if (((RegisterByte > 0x65) && (RegisterByte < 0x69)) ||
 584                                       (RegisterByte == 0x2C) || (RegisterByte == 0xF0))
 585                             {
 586                                 /* Indicates L1 cache line of 64 bytes */
 587                                 KePrefetchNTAGranularity = 64;
 588                             }
 589                             else if (RegisterByte == 0xF1)
 590                             {
 591                                 /* Indicates L1 cache line of 128 bytes */
 592                                 KePrefetchNTAGranularity = 128;
 593                             }
 594                             else if (((RegisterByte >= 0x4A) && (RegisterByte <= 0x4C)) ||
 595                                       (RegisterByte == 0x78) ||
 596                                       (RegisterByte == 0x7D) ||
 597                                       (RegisterByte == 0x7F) ||
 598                                       (RegisterByte == 0x86) ||
 599                                       (RegisterByte == 0x87))
 600                             {
 601                                 /* Set minimum cache line size */
 602                                 if (CacheLine < 64) CacheLine = 64;
 603
 604                                 /* Hard-code size/associativity */
 605                                 switch (RegisterByte)
 606                                 {
 607                                     case 0x4A:
 608                                         Size = 4 * 1024 * 1024;
 609                                         Associativity = 8;
 610                                         break;
 611
 612                                     case 0x4B:
 613                                         Size = 6 * 1024 * 1024;
 614                                         Associativity = 12;
 615                                         break;
 616
 617                                     case 0x4C:
 618                                         Size = 8 * 1024 * 1024;
 619                                         Associativity = 16;
 620                                         break;
 621
 622                                     case 0x78:
 623                                         Size = 1 * 1024 * 1024;
 624                                         Associativity = 4;
 625                                         break;
 626
 627                                     case 0x7D:
 628                                         Size = 2 * 1024 * 1024;
 629                                         Associativity = 8;
 630                                         break;
 631
 632                                     case 0x7F:
 633                                         Size = 512 * 1024;
 634                                         Associativity = 2;
 635                                         break;
 636
 637                                     case 0x86:
 638                                         Size = 512 * 1024;
 639                                         Associativity = 4;
 640                                         break;
 641
 642                                     case 0x87:
 643                                         Size = 1 * 1024 * 1024;
 644                                         Associativity = 8;
 645                                         break;
 646
 647                                     default:
 648                                         Size = 0;
 649                                         break;
 650                                 }
 651
 652                                 /* Check if this cache is bigger than the last */
 653                                 if ((Size / Associativity) > CurrentSize)
 654                                 {
 655                                     /* Set the L2 Cache Size and Associativity */
 656                                     CurrentSize = Size / Associativity;
 657                                     Pcr->SecondLevelCacheSize = Size;
 658                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 659                                 }
 660                             }
 661                         }
 662                     }
 663                 } while (--CacheRequests);
 664             }
 665             break;
 666
 667         case CPU_AMD:
 668
 669             /* Check if we support CPUID 0x80000005 */
 670             CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 671             if (Data[0] >= 0x80000006)
 672             {
 673                 /* Get L1 size first */
 674                 CPUID(0x80000005, &Data[0], &Data[1], &Data[2], &Data[3]);
 675                 KePrefetchNTAGranularity = Data[2] & 0xFF;
 676
 677                 /* Check if we support CPUID 0x80000006 */
 678                 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 679                 if (Data[0] >= 0x80000006)
 680                 {
 681                     /* Get 2nd level cache and tlb size */
 682                     CPUID(0x80000006, &Data[0], &Data[1], &Data[2], &Data[3]);
 683
 684                     /* Cache line size */
 685                     CacheLine = Data[2] & 0xFF;
 686
 687                     /* Hardcode associativity */
 688                     RegisterByte = Data[2] >> 12;
 689                     switch (RegisterByte)
 690                     {
 691                         case 2:
 692                             Associativity = 2;
 693                             break;
 694
 695                         case 4:
 696                             Associativity = 4;
 697                             break;
 698
 699                         case 6:
 700                             Associativity = 8;
 701                             break;
 702
 703                         case 8:
 704                         case 15:
 705                             Associativity = 16;
 706                             break;
 707
 708                         default:
 709                             Associativity = 1;
 710                             break;
 711                     }
 712
 713                     /* Compute size */
 714                     Size = (Data[2] >> 16) << 10;
 715
 716                     /* Hack for Model 6, Steping 300 */
 717                     if ((KeGetCurrentPrcb()->CpuType == 6) &&
 718                         (KeGetCurrentPrcb()->CpuStep == 0x300))
 719                     {
 720                         /* Stick 64K in there */
 721                         Size = 64 * 1024;
 722                     }
 723
 724                     /* Set the L2 Cache Size and associativity */
 725                     Pcr->SecondLevelCacheSize = Size;
 726                     Pcr->SecondLevelCacheAssociativity = Associativity;
 727                 }
 728             }
 729             break;
 730
 731         case CPU_CYRIX:
 732         case CPU_TRANSMETA:
 733         case CPU_CENTAUR:
 734         case CPU_RISE:
 735
 736             /* FIXME */
 737             break;
 738     }
 739
 740     /* Set the cache line */
 741     if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
 742     DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
 743             KePrefetchNTAGranularity,
 744             Pcr->SecondLevelCacheSize,
 745             KeLargestCacheLine,
 746             Pcr->SecondLevelCacheAssociativity);
 747 }
 748
 749 VOID
 750 NTAPI
 751 KiSetCR0Bits(VOID)
 752 {
 753     ULONG Cr0;
 754
 755     /* Save current CR0 */
 756     Cr0 = __readcr0();
 757
 758     /* If this is a 486, enable Write-Protection */
 759     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 760
 761     /* Set new Cr0 */
 762     __writecr0(Cr0);
 763 }
 764
 765 VOID
 766 NTAPI
 767 KiInitializeTSS2(IN PKTSS Tss,
 768                  IN PKGDTENTRY TssEntry OPTIONAL)
 769 {
 770     PUCHAR p;
 771
 772     /* Make sure the GDT Entry is valid */
 773     if (TssEntry)
 774     {
 775         /* Set the Limit */
 776         TssEntry->LimitLow = sizeof(KTSS) - 1;
 777         TssEntry->HighWord.Bits.LimitHi = 0;
 778     }
 779
 780     /* Now clear the I/O Map */
 781     ASSERT(IOPM_COUNT == 1);
 782     RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
 783
 784     /* Initialize Interrupt Direction Maps */
 785     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 786     RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
 787
 788     /* Add DPMI support for interrupts */
 789     p[0] = 4;
 790     p[3] = 0x18;
 791     p[4] = 0x18;
 792
 793     /* Initialize the default Interrupt Direction Map */
 794     p = Tss->IntDirectionMap;
 795     RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
 796
 797     /* Add DPMI support */
 798     p[0] = 4;
 799     p[3] = 0x18;
 800     p[4] = 0x18;
 801 }
 802
 803 VOID
 804 NTAPI
 805 KiInitializeTSS(IN PKTSS Tss)
 806 {
 807     /* Set an invalid map base */
 808     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 809
 810     /* Disable traps during Task Switches */
 811     Tss->Flags = 0;
 812
 813     /* Set LDT and Ring 0 SS */
 814     Tss->LDT = 0;
 815     Tss->Ss0 = KGDT_R0_DATA;
 816 }
 817
 818 VOID
 819 FASTCALL
 820 Ki386InitializeTss(IN PKTSS Tss,
 821                    IN PKIDTENTRY Idt,
 822                    IN PKGDTENTRY Gdt)
 823 {
 824     PKGDTENTRY TssEntry, TaskGateEntry;
 825
 826     /* Initialize the boot TSS. */
 827     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 828     TssEntry->HighWord.Bits.Type = I386_TSS;
 829     TssEntry->HighWord.Bits.Pres = 1;
 830     TssEntry->HighWord.Bits.Dpl = 0;
 831     KiInitializeTSS2(Tss, TssEntry);
 832     KiInitializeTSS(Tss);
 833
 834     /* Load the task register */
 835     Ke386SetTr(KGDT_TSS);
 836
 837     /* Setup the Task Gate for Double Fault Traps */
 838     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 839     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 840     TaskGateEntry->HighWord.Bits.Pres = 1;
 841     TaskGateEntry->HighWord.Bits.Dpl = 0;
 842     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 843
 844     /* Initialize the TSS used for handling double faults. */
 845     Tss = (PKTSS)KiDoubleFaultTSS;
 846     KiInitializeTSS(Tss);
 847     Tss->CR3 = __readcr3();
 848     Tss->Esp0 = KiDoubleFaultStack;
 849     Tss->Esp = KiDoubleFaultStack;
 850     Tss->Eip = PtrToUlong(KiTrap08);
 851     Tss->Cs = KGDT_R0_CODE;
 852     Tss->Fs = KGDT_R0_PCR;
 853     Tss->Ss = Ke386GetSs();
 854     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 855     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 856
 857     /* Setup the Double Trap TSS entry in the GDT */
 858     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 859     TssEntry->HighWord.Bits.Type = I386_TSS;
 860     TssEntry->HighWord.Bits.Pres = 1;
 861     TssEntry->HighWord.Bits.Dpl = 0;
 862     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 863     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 864     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 865     TssEntry->LimitLow = KTSS_IO_MAPS;
 866
 867     /* Now setup the NMI Task Gate */
 868     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 869     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 870     TaskGateEntry->HighWord.Bits.Pres = 1;
 871     TaskGateEntry->HighWord.Bits.Dpl = 0;
 872     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 873
 874     /* Initialize the actual TSS */
 875     Tss = (PKTSS)KiNMITSS;
 876     KiInitializeTSS(Tss);
 877     Tss->CR3 = __readcr3();
 878     Tss->Esp0 = KiDoubleFaultStack;
 879     Tss->Esp = KiDoubleFaultStack;
 880     Tss->Eip = PtrToUlong(KiTrap02);
 881     Tss->Cs = KGDT_R0_CODE;
 882     Tss->Fs = KGDT_R0_PCR;
 883     Tss->Ss = Ke386GetSs();
 884     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 885     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 886
 887     /* And its associated TSS Entry */
 888     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 889     TssEntry->HighWord.Bits.Type = I386_TSS;
 890     TssEntry->HighWord.Bits.Pres = 1;
 891     TssEntry->HighWord.Bits.Dpl = 0;
 892     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 893     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 894     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 895     TssEntry->LimitLow = KTSS_IO_MAPS;
 896 }
 897
 898 VOID
 899 NTAPI
 900 KeFlushCurrentTb(VOID)
 901 {
 902     /* Flush the TLB by resetting CR3 */
 903     __writecr3(__readcr3());
 904 }
 905
 906 VOID
 907 NTAPI
 908 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 909 {
 910     PKGDTENTRY TssEntry;
 911
 912     //
 913     // Restore the CR registers
 914     //
 915     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 916     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 917     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 918     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 919
 920     //
 921     // Restore the DR registers
 922     //
 923     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 924     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 925     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 926     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 927     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 928     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 929
 930     //
 931     // Restore GDT and IDT
 932     //
 933     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 934     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 935
 936     //
 937     // Clear the busy flag so we don't crash if we reload the same selector
 938     //
 939     TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
 940                             ProcessorState->SpecialRegisters.Tr);
 941     TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
 942
 943     //
 944     // Restore TSS and LDT
 945     //
 946     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 947     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 948 }
 949
 950 VOID
 951 NTAPI
 952 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 953 {
 954     /* Save the CR registers */
 955     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 956     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 957     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 958     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 959                                            __readcr4() : 0;
 960
 961     /* Save the DR registers */
 962     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 963     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 964     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 965     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 966     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 967     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 968     __writedr(7, 0);
 969
 970     /* Save GDT, IDT, LDT and TSS */
 971     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 972     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 973     ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
 974     ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
 975 }
 976
 977 VOID
 978 NTAPI
 979 KiInitializeMachineType(VOID)
 980 {
 981     /* Set the Machine Type we got from NTLDR */
 982     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 983 }
 984
 985 ULONG_PTR
 986 NTAPI
 987 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 988 {
 989     /* Set CS and ESP */
 990     WRMSR(0x174, KGDT_R0_CODE);
 991     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
 992
 993     /* Set LSTAR */
 994     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
 995     return 0;
 996 }
 997
 998 VOID
 999 NTAPI
1000 KiDisableFastSyscallReturn(VOID)
1001 {
1002     /* Was it applied? */
1003     if (KiSystemCallExitAdjusted)
1004     {
1005         /* Restore the original value */
1006         KiSystemCallExitBranch[1] = KiSystemCallExitBranch[1] - KiSystemCallExitAdjusted;
1007
1008         /* It's not adjusted anymore */
1009         KiSystemCallExitAdjusted = FALSE;
1010     }
1011 }
1012
1013 VOID
1014 NTAPI
1015 KiEnableFastSyscallReturn(VOID)
1016 {
1017     /* Check if the patch has already been done */
1018     if ((KiSystemCallExitAdjusted == KiSystemCallExitAdjust) &&
1019         (KiFastCallCopyDoneOnce))
1020     {
1021         return;
1022     }
1023
1024     /* Make sure the offset is within the distance of a Jxx SHORT */
1025     if ((KiSystemCallExitBranch[1] - KiSystemCallExitAdjust) < 0x80)
1026     {
1027         /* Remove any existing code patch */
1028         KiDisableFastSyscallReturn();
1029
1030         /* We should have a JNZ there */
1031         ASSERT(KiSystemCallExitBranch[0] == 0x75);
1032
1033         /* Do the patch */
1034         KiSystemCallExitAdjusted = KiSystemCallExitAdjust;
1035         KiSystemCallExitBranch[1] -= KiSystemCallExitAdjusted;
1036
1037         /* Remember that we've done it */
1038         KiFastCallCopyDoneOnce = TRUE;
1039     }
1040     else
1041     {
1042         /* This shouldn't happen unless we've messed the macros up */
1043         DPRINT1("Your compiled kernel is broken!\n");
1044         DbgBreakPoint();
1045     }
1046 }
1047
1048 VOID
1049 NTAPI
1050 KiRestoreFastSyscallReturnState(VOID)
1051 {
1052     /* Check if the CPU Supports fast system call */
1053     if (KeFeatureBits & KF_FAST_SYSCALL)
1054     {
1055         /* Check if it has been disabled */
1056         if (!KiFastSystemCallDisable)
1057         {
1058             /* KiSystemCallExit2 should come BEFORE KiSystemCallExit */
1059             ASSERT(KiSystemCallExit2 < KiSystemCallExit);
1060
1061             /* It's enabled, so we'll have to do a code patch */
1062             KiSystemCallExitAdjust = KiSystemCallExit - KiSystemCallExit2;
1063         }
1064         else
1065         {
1066             /* Disable fast system call */
1067             KeFeatureBits &= ~KF_FAST_SYSCALL;
1068         }
1069     }
1070
1071     /* Now check if all CPUs support fast system call, and the registry allows it */
1072     if (KeFeatureBits & KF_FAST_SYSCALL)
1073     {
1074         /* Do an IPI to enable it */
1075         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
1076     }
1077
1078     /* Perform the code patch that is required */
1079     KiEnableFastSyscallReturn();
1080 }
1081
1082 ULONG_PTR
1083 NTAPI
1084 Ki386EnableDE(IN ULONG_PTR Context)
1085 {
1086     /* Enable DE */
1087     __writecr4(__readcr4() | CR4_DE);
1088     return 0;
1089 }
1090
1091 ULONG_PTR
1092 NTAPI
1093 Ki386EnableFxsr(IN ULONG_PTR Context)
1094 {
1095     /* Enable FXSR */
1096     __writecr4(__readcr4() | CR4_FXSR);
1097     return 0;
1098 }
1099
1100 ULONG_PTR
1101 NTAPI
1102 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
1103 {
1104     PKIDTENTRY IdtEntry;
1105
1106     /* Get the IDT Entry for Interrupt 0x13 */
1107     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
1108
1109     /* Set it up */
1110     IdtEntry->Selector = KGDT_R0_CODE;
1111     IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
1112     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
1113     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
1114     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
1115     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
1116
1117     /* Enable XMMI exceptions */
1118     __writecr4(__readcr4() | CR4_XMMEXCPT);
1119     return 0;
1120 }
1121
1122 VOID
1123 NTAPI
1124 KiI386PentiumLockErrataFixup(VOID)
1125 {
1126     KDESCRIPTOR IdtDescriptor;
1127     PKIDTENTRY NewIdt, NewIdt2;
1128
1129     /* Allocate memory for a new IDT */
1130     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
1131
1132     /* Put everything after the first 7 entries on a new page */
1133     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
1134
1135     /* Disable interrupts */
1136     _disable();
1137
1138     /* Get the current IDT and copy it */
1139     __sidt(&IdtDescriptor.Limit);
1140     RtlCopyMemory(NewIdt2,
1141                   (PVOID)IdtDescriptor.Base,
1142                   IdtDescriptor.Limit + 1);
1143     IdtDescriptor.Base = (ULONG)NewIdt2;
1144
1145     /* Set the new IDT */
1146     __lidt(&IdtDescriptor.Limit);
1147     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
1148
1149     /* Restore interrupts */
1150     _enable();
1151
1152     /* Set the first 7 entries as read-only to produce a fault */
1153     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
1154 }
1155
1156 BOOLEAN
1157 NTAPI
1158 KeDisableInterrupts(VOID)
1159 {
1160     ULONG Flags;
1161     BOOLEAN Return;
1162
1163     /* Get EFLAGS and check if the interrupt bit is set */
1164     Flags = __readeflags();
1165     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
1166
1167     /* Disable interrupts */
1168     _disable();
1169     return Return;
1170 }
1171
1172 BOOLEAN
1173 NTAPI
1174 KeInvalidateAllCaches(VOID)
1175 {
1176     /* Only supported on Pentium Pro and higher */
1177     if (KeI386CpuType < 6) return FALSE;
1178
1179     /* Invalidate all caches */
1180     __wbinvd();
1181     return TRUE;
1182 }
1183
1184 VOID
1185 FASTCALL
1186 KeZeroPages(IN PVOID Address,
1187             IN ULONG Size)
1188 {
1189     /* Not using XMMI in this routine */
1190     RtlZeroMemory(Address, Size);
1191 }
1192
1193 VOID
1194 NTAPI
1195 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
1196                      IN PKEXCEPTION_FRAME ExceptionFrame)
1197 {
1198     PKPRCB Prcb = KeGetCurrentPrcb();
1199
1200     //
1201     // Save full context
1202     //
1203     Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
1204                                                      CONTEXT_DEBUG_REGISTERS;
1205     KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
1206
1207     //
1208     // Save control registers
1209     //
1210     KiSaveProcessorControlState(&Prcb->ProcessorState);
1211 }
1212
1213 BOOLEAN
1214 NTAPI
1215 KiIsNpxPresent(VOID)
1216 {
1217     ULONG Cr0;
1218     USHORT Magic;
1219
1220     /* Set magic */
1221     Magic = 0xFFFF;
1222
1223     /* Read CR0 and mask out FPU flags */
1224     Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
1225
1226     /* Store on FPU stack */
1227     asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
1228
1229     /* Magic should now be cleared */
1230     if (Magic & 0xFF)
1231     {
1232         /* You don't have an FPU -- enable emulation for now */
1233         __writecr0(Cr0 | CR0_EM | CR0_TS);
1234         return FALSE;
1235     }
1236
1237     /* You have an FPU, enable it */
1238     Cr0 |= CR0_ET;
1239
1240     /* Enable INT 16 on 486 and higher */
1241     if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
1242
1243     /* Set FPU state */
1244     __writecr0(Cr0 | CR0_EM | CR0_TS);
1245     return TRUE;
1246 }
1247
1248 BOOLEAN
1249 NTAPI
1250 KiIsNpxErrataPresent(VOID)
1251 {
1252     BOOLEAN ErrataPresent;
1253     ULONG Cr0;
1254     volatile double Value1, Value2;
1255
1256     /* Disable interrupts */
1257     _disable();
1258
1259     /* Read CR0 and remove FPU flags */
1260     Cr0 = __readcr0();
1261     __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1262
1263     /* Initialize FPU state */
1264     asm volatile ("fninit");
1265
1266     /* Multiply the magic values and divide, we should get the result back */
1267     Value1 = 4195835.0;
1268     Value2 = 3145727.0;
1269     ErrataPresent = (Value1 * Value2 / 3145727.0) != 4195835.0;
1270
1271     /* Restore CR0 */
1272     __writecr0(Cr0);
1273
1274     /* Enable interrupts */
1275     _enable();
1276
1277     /* Return if there's an errata */
1278     return ErrataPresent;
1279 }
1280
1281 NTAPI
1282 VOID
1283 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
1284 {
1285     ULONG EFlags, Cr0;
1286     PKTHREAD Thread, NpxThread;
1287     PFX_SAVE_AREA FxSaveArea;
1288
1289     /* Save volatiles and disable interrupts */
1290     EFlags = __readeflags();
1291     _disable();
1292
1293     /* Save the PCR and get the current thread */
1294     Thread = KeGetCurrentThread();
1295
1296     /* Check if we're already loaded */
1297     if (Thread->NpxState != NPX_STATE_LOADED)
1298     {
1299         /* If there's nothing to load, quit */
1300         if (!SaveArea) return;
1301
1302         /* Need FXSR support for this */
1303         ASSERT(KeI386FxsrPresent == TRUE);
1304
1305         /* Check for sane CR0 */
1306         Cr0 = __readcr0();
1307         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1308         {
1309             /* Mask out FPU flags */
1310             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1311         }
1312
1313         /* Get the NPX thread and check its FPU state */
1314         NpxThread = KeGetCurrentPrcb()->NpxThread;
1315         if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
1316         {
1317             /* Get the FX frame and store the state there */
1318             FxSaveArea = KiGetThreadNpxArea(NpxThread);
1319             Ke386FxSave(FxSaveArea);
1320
1321             /* NPX thread has lost its state */
1322             NpxThread->NpxState = NPX_STATE_NOT_LOADED;
1323         }
1324
1325         /* Now load NPX state from the NPX area */
1326         FxSaveArea = KiGetThreadNpxArea(Thread);
1327         Ke386FxStore(FxSaveArea);
1328     }
1329     else
1330     {
1331         /* Check for sane CR0 */
1332         Cr0 = __readcr0();
1333         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1334         {
1335             /* Mask out FPU flags */
1336             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1337         }
1338
1339         /* Get FX frame */
1340         FxSaveArea = KiGetThreadNpxArea(Thread);
1341         Thread->NpxState = NPX_STATE_NOT_LOADED;
1342
1343         /* Save state if supported by CPU */
1344         if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
1345     }
1346
1347     /* Now save the FN state wherever it was requested */
1348     if (SaveArea) Ke386FnSave(SaveArea);
1349
1350     /* Clear NPX thread */
1351     KeGetCurrentPrcb()->NpxThread = NULL;
1352
1353     /* Add the CR0 from the NPX frame */
1354     Cr0 |= NPX_STATE_NOT_LOADED;
1355     Cr0 |= FxSaveArea->Cr0NpxState;
1356     __writecr0(Cr0);
1357
1358     /* Restore interrupt state */
1359     __writeeflags(EFlags);
1360 }
1361
1362 /* PUBLIC FUNCTIONS **********************************************************/
1363
1364 /*
1365  * @implemented
1366  */
1367 VOID
1368 NTAPI
1369 KiCoprocessorError(VOID)
1370 {
1371     PFX_SAVE_AREA NpxArea;
1372
1373     /* Get the FPU area */
1374     NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
1375
1376     /* Set CR0_TS */
1377     NpxArea->Cr0NpxState = CR0_TS;
1378     __writecr0(__readcr0() | CR0_TS);
1379 }
1380
1381 /*
1382  * @implemented
1383  */
1384 NTSTATUS
1385 NTAPI
1386 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
1387 {
1388     PFNSAVE_FORMAT FpState;
1389     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
1390     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1391
1392     /* check if we are doing software emulation */
1393     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
1394
1395     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1396     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1397
1398     *((PVOID *) Save) = FpState;
1399 #ifdef __GNUC__
1400     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1401 #else
1402     __asm
1403     {
1404         fnsave [FpState]
1405     };
1406 #endif
1407
1408     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1409     return STATUS_SUCCESS;
1410 }
1411
1412 /*
1413  * @implemented
1414  */
1415 NTSTATUS
1416 NTAPI
1417 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1418 {
1419     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1420     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1421     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1422
1423 #ifdef __GNUC__
1424     asm volatile("fnclex\n\t");
1425     asm volatile("frstor %0\n\t" : "=m" (*FpState));
1426 #else
1427     __asm
1428     {
1429         fnclex
1430         frstor [FpState]
1431     };
1432 #endif
1433
1434     ExFreePool(FpState);
1435     return STATUS_SUCCESS;
1436 }
1437
1438 /*
1439  * @implemented
1440  */
1441 ULONG
1442 NTAPI
1443 KeGetRecommendedSharedDataAlignment(VOID)
1444 {
1445     /* Return the global variable */
1446     return KeLargestCacheLine;
1447 }
1448
1449 VOID
1450 NTAPI
1451 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1452                       IN PVOID Ignored1,
1453                       IN PVOID Ignored2,
1454                       IN PVOID Ignored3)
1455 {
1456     /* Signal this packet as done */
1457     KiIpiSignalPacketDone(PacketContext);
1458
1459     /* Flush the TB for the Current CPU */
1460     KeFlushCurrentTb();
1461 }
1462
1463 /*
1464  * @implemented
1465  */
1466 VOID
1467 NTAPI
1468 KeFlushEntireTb(IN BOOLEAN Invalid,
1469                 IN BOOLEAN AllProcessors)
1470 {
1471     KIRQL OldIrql;
1472 #ifdef CONFIG_SMP
1473     KAFFINITY TargetAffinity;
1474     PKPRCB Prcb = KeGetCurrentPrcb();
1475 #endif
1476
1477     /* Raise the IRQL for the TB Flush */
1478     OldIrql = KeRaiseIrqlToSynchLevel();
1479
1480 #ifdef CONFIG_SMP
1481     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1482
1483     /* Get the current processor affinity, and exclude ourselves */
1484     TargetAffinity = KeActiveProcessors;
1485     TargetAffinity &= ~Prcb->SetMember;
1486
1487     /* Make sure this is MP */
1488     if (TargetAffinity)
1489     {
1490         /* Send an IPI TB flush to the other processors */
1491         KiIpiSendPacket(TargetAffinity,
1492                         KiFlushTargetEntireTb,
1493                         NULL,
1494                         0,
1495                         NULL);
1496     }
1497 #endif
1498
1499     /* Flush the TB for the Current CPU, and update the flush stamp */
1500     KeFlushCurrentTb();
1501
1502 #ifdef CONFIG_SMP
1503     /* If this is MP, wait for the other processors to finish */
1504     if (TargetAffinity)
1505     {
1506         /* Sanity check */
1507         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1508
1509         /* FIXME: TODO */
1510         ASSERTMSG("Not yet implemented\n", FALSE);
1511     }
1512 #endif
1513
1514     /* Update the flush stamp and return to original IRQL */
1515     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1516     KeLowerIrql(OldIrql);
1517 }
1518
1519 /*
1520  * @implemented
1521  */
1522 VOID
1523 NTAPI
1524 KeSetDmaIoCoherency(IN ULONG Coherency)
1525 {
1526     /* Save the coherency globally */
1527     KiDmaIoCoherency = Coherency;
1528 }
1529
1530 /*
1531  * @implemented
1532  */
1533 KAFFINITY
1534 NTAPI
1535 KeQueryActiveProcessors(VOID)
1536 {
1537     PAGED_CODE();
1538
1539     /* Simply return the number of active processors */
1540     return KeActiveProcessors;
1541 }
1542
1543 /*
1544  * @implemented
1545  */
1546 VOID
1547 __cdecl
1548 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1549 {
1550     /* Capture the context */
1551     RtlCaptureContext(&State->ContextFrame);
1552
1553     /* Capture the control state */
1554     KiSaveProcessorControlState(State);
1555 }