ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The TSS to use for Double Fault Traps (INT 0x9) */
  18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  19
  20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  21 UCHAR KiNMITSS[KTSS_IO_MAPS];
  22
  23 /* CPU Features and Flags */
  24 ULONG KeI386CpuType;
  25 ULONG KeI386CpuStep;
  26 ULONG KeProcessorArchitecture;
  27 ULONG KeProcessorLevel;
  28 ULONG KeProcessorRevision;
  29 ULONG KeFeatureBits;
  30 ULONG KiFastSystemCallDisable;
  31 ULONG KeI386NpxPresent = 0;
  32 ULONG KiMXCsrMask = 0;
  33 ULONG MxcsrFeatureMask = 0;
  34 ULONG KeI386XMMIPresent = 0;
  35 ULONG KeI386FxsrPresent = 0;
  36 ULONG KeI386MachineType;
  37 ULONG Ke386Pae = FALSE;
  38 ULONG Ke386NoExecute = FALSE;
  39 ULONG KeLargestCacheLine = 0x40;
  40 ULONG KeDcacheFlushCount = 0;
  41 ULONG KeIcacheFlushCount = 0;
  42 ULONG KiDmaIoCoherency = 0;
  43 ULONG KePrefetchNTAGranularity = 32;
  44 CHAR KeNumberProcessors;
  45 KAFFINITY KeActiveProcessors = 1;
  46 BOOLEAN KiI386PentiumLockErrataPresent;
  47 BOOLEAN KiSMTProcessorsPresent;
  48
  49 /* The distance between SYSEXIT and IRETD return modes */
  50 UCHAR KiSystemCallExitAdjust;
  51
  52 /* The offset that was applied -- either 0 or the value above */
  53 UCHAR KiSystemCallExitAdjusted;
  54
  55 /* Whether the adjustment was already done once */
  56 BOOLEAN KiFastCallCopyDoneOnce;
  57
  58 /* Flush data */
  59 volatile LONG KiTbFlushTimeStamp;
  60
  61 /* CPU Signatures */
  62 static const CHAR CmpIntelID[]       = "GenuineIntel";
  63 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  64 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  65 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  66 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  67 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  68
  69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  70
  71 VOID
  72 NTAPI
  73 CPUID(IN ULONG InfoType,
  74       OUT PULONG CpuInfoEax,
  75       OUT PULONG CpuInfoEbx,
  76       OUT PULONG CpuInfoEcx,
  77       OUT PULONG CpuInfoEdx)
  78 {
  79     ULONG CpuInfo[4];
  80
  81     /* Perform the CPUID Operation */
  82     __cpuid((int*)CpuInfo, InfoType);
  83
  84     /* Return the results */
  85     *CpuInfoEax = CpuInfo[0];
  86     *CpuInfoEbx = CpuInfo[1];
  87     *CpuInfoEcx = CpuInfo[2];
  88     *CpuInfoEdx = CpuInfo[3];
  89 }
  90
  91 VOID
  92 NTAPI
  93 WRMSR(IN ULONG Register,
  94       IN LONGLONG Value)
  95 {
  96    /* Write to the MSR */
  97     __writemsr(Register, Value);
  98 }
  99
 100 LONGLONG
 101 FASTCALL
 102 RDMSR(IN ULONG Register)
 103 {
 104     /* Read from the MSR */
 105     return __readmsr(Register);
 106 }
 107
 108 /* NSC/Cyrix CPU configuration register index */
 109 #define CX86_CCR1 0xc1
 110
 111 /* NSC/Cyrix CPU indexed register access macros */
 112 #define getCx86(reg) ({ WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22,(reg)); READ_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23); })
 113
 114 #define setCx86(reg, data) do { \
 115    WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22,(reg)); \
 116    WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23,(data)); \
 117 } while (0)
 118
 119 /* FUNCTIONS *****************************************************************/
 120
 121 VOID
 122 NTAPI
 123 KiSetProcessorType(VOID)
 124 {
 125     ULONG EFlags, NewEFlags;
 126     ULONG Reg, Dummy;
 127     ULONG Stepping, Type;
 128
 129     /* Start by assuming no CPUID data */
 130     KeGetCurrentPrcb()->CpuID = 0;
 131
 132     /* Save EFlags */
 133     EFlags = __readeflags();
 134
 135     /* XOR out the ID bit and update EFlags */
 136     NewEFlags = EFlags ^ EFLAGS_ID;
 137     __writeeflags(NewEFlags);
 138
 139     /* Get them back and see if they were modified */
 140     NewEFlags = __readeflags();
 141     if (NewEFlags != EFlags)
 142     {
 143         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 144         EFlags |= EFLAGS_ID;
 145         __writeeflags(EFlags);
 146
 147         /* Peform CPUID 0 to see if CPUID 1 is supported */
 148         CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
 149         if (Reg > 0)
 150         {
 151             /* Do CPUID 1 now */
 152             CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
 153
 154             /*
 155              * Get the Stepping and Type. The stepping contains both the
 156              * Model and the Step, while the Type contains the returned Type.
 157              * We ignore the family.
 158              *
 159              * For the stepping, we convert this: zzzzzzxy into this: x0y
 160              */
 161             Stepping = Reg & 0xF0;
 162             Stepping <<= 4;
 163             Stepping += (Reg & 0xFF);
 164             Stepping &= 0xF0F;
 165             Type = Reg & 0xF00;
 166             Type >>= 8;
 167
 168             /* Save them in the PRCB */
 169             KeGetCurrentPrcb()->CpuID = TRUE;
 170             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 171             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 172         }
 173         else
 174         {
 175             DPRINT1("CPUID Support lacking\n");
 176         }
 177     }
 178     else
 179     {
 180         DPRINT1("CPUID Support lacking\n");
 181     }
 182
 183     /* Restore EFLAGS */
 184     __writeeflags(EFlags);
 185 }
 186
 187 ULONG
 188 NTAPI
 189 KiGetCpuVendor(VOID)
 190 {
 191     PKPRCB Prcb = KeGetCurrentPrcb();
 192     ULONG Vendor[5];
 193     ULONG Temp;
 194
 195     /* Assume no Vendor ID and fail if no CPUID Support. */
 196     Prcb->VendorString[0] = 0;
 197     if (!Prcb->CpuID) return 0;
 198
 199     /* Get the Vendor ID and null-terminate it */
 200     CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
 201     Vendor[4] = 0;
 202
 203     /* Re-arrange vendor string */
 204     Temp = Vendor[2];
 205     Vendor[2] = Vendor[3];
 206     Vendor[3] = Temp;
 207
 208     /* Copy it to the PRCB and null-terminate it again */
 209     RtlCopyMemory(Prcb->VendorString,
 210                   &Vendor[1],
 211                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 212     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 213
 214     /* Now check the CPU Type */
 215     if (!strcmp(Prcb->VendorString, CmpIntelID))
 216     {
 217         return CPU_INTEL;
 218     }
 219     else if (!strcmp(Prcb->VendorString, CmpAmdID))
 220     {
 221         return CPU_AMD;
 222     }
 223     else if (!strcmp(Prcb->VendorString, CmpCyrixID))
 224     {
 225         DPRINT1("Cyrix CPU support not fully tested!\n");
 226         return CPU_CYRIX;
 227     }
 228     else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
 229     {
 230         DPRINT1("Transmeta CPU support not fully tested!\n");
 231         return CPU_TRANSMETA;
 232     }
 233     else if (!strcmp(Prcb->VendorString, CmpCentaurID))
 234     {
 235         DPRINT1("Centaur CPU support not fully tested!\n");
 236         return CPU_CENTAUR;
 237     }
 238     else if (!strcmp(Prcb->VendorString, CmpRiseID))
 239     {
 240         DPRINT1("Rise CPU support not fully tested!\n");
 241         return CPU_RISE;
 242     }
 243
 244     /* Invalid CPU */
 245     return 0;
 246 }
 247
 248 ULONG
 249 NTAPI
 250 KiGetFeatureBits(VOID)
 251 {
 252     PKPRCB Prcb = KeGetCurrentPrcb();
 253     ULONG Vendor;
 254     ULONG FeatureBits = KF_WORKING_PTE;
 255     ULONG Reg[4], Dummy, Ccr1;
 256     BOOLEAN ExtendedCPUID = TRUE;
 257     ULONG CpuFeatures = 0;
 258
 259     /* Get the Vendor ID */
 260     Vendor = KiGetCpuVendor();
 261
 262     /* Make sure we got a valid vendor ID at least. */
 263     if (!Vendor) return FeatureBits;
 264
 265     /* Get the CPUID Info. Features are in Reg[3]. */
 266     CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
 267
 268     /* Set the initial APIC ID */
 269     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 270
 271     switch (Vendor)
 272     {
 273         /* Intel CPUs */
 274         case CPU_INTEL:
 275
 276             /* Check if it's a P6 */
 277             if (Prcb->CpuType == 6)
 278             {
 279                 /* Perform the special sequence to get the MicroCode Signature */
 280                 WRMSR(0x8B, 0);
 281                 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
 282                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 283             }
 284             else if (Prcb->CpuType == 5)
 285             {
 286                 /* On P5, enable workaround for the LOCK errata. */
 287                 KiI386PentiumLockErrataPresent = TRUE;
 288             }
 289
 290             /* Check for broken P6 with bad SMP PTE implementation */
 291             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 292                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 293             {
 294                 /* Remove support for correct PTE support. */
 295                 FeatureBits &= ~KF_WORKING_PTE;
 296             }
 297
 298             /* Check if the CPU is too old to support SYSENTER */
 299             if ((Prcb->CpuType < 6) ||
 300                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 301             {
 302                 /* Disable it */
 303                 Reg[3] &= ~0x800;
 304             }
 305
 306             /* Set the current features */
 307             CpuFeatures = Reg[3];
 308
 309             break;
 310
 311         /* AMD CPUs */
 312         case CPU_AMD:
 313
 314             /* Check if this is a K5 or K6. (family 5) */
 315             if ((Reg[0] & 0x0F00) == 0x0500)
 316             {
 317                 /* Get the Model Number */
 318                 switch (Reg[0] & 0x00F0)
 319                 {
 320                     /* Model 1: K5 - 5k86 (initial models) */
 321                     case 0x0010:
 322
 323                         /* Check if this is Step 0 or 1. They don't support PGE */
 324                         if ((Reg[0] & 0x000F) > 0x03) break;
 325
 326                     /* Model 0: K5 - SSA5 */
 327                     case 0x0000:
 328
 329                         /* Model 0 doesn't support PGE at all. */
 330                         Reg[3] &= ~0x2000;
 331                         break;
 332
 333                     /* Model 8: K6-2 */
 334                     case 0x0080:
 335
 336                         /* K6-2, Step 8 and over have support for MTRR. */
 337                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 338                         break;
 339
 340                     /* Model 9: K6-III
 341                        Model D: K6-2+, K6-III+ */
 342                     case 0x0090:
 343                     case 0x00D0:
 344
 345                         FeatureBits |= KF_AMDK6MTRR;
 346                         break;
 347                 }
 348             }
 349             else if((Reg[0] & 0x0F00) < 0x0500)
 350             {
 351                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 352                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 353
 354                 /* They also don't support advanced CPUID functions. */
 355                 ExtendedCPUID = FALSE;
 356             }
 357
 358             /* Set the current features */
 359             CpuFeatures = Reg[3];
 360
 361             break;
 362
 363         /* Cyrix CPUs */
 364         case CPU_CYRIX:
 365
 366             /* Workaround the "COMA" bug on 6x family of Cyrix CPUs */
 367             if (Prcb->CpuType == 6 &&
 368                 Prcb->CpuStep <= 1)
 369             {
 370                 /* Get CCR1 value */
 371                 Ccr1 = getCx86(CX86_CCR1);
 372
 373                 /* Enable the NO_LOCK bit */
 374                 Ccr1 |= 0x10;
 375
 376                 /* Set the new CCR1 value */
 377                 setCx86(CX86_CCR1, Ccr1);
 378             }
 379
 380             /* Set the current features */
 381             CpuFeatures = Reg[3];
 382
 383             break;
 384
 385         /* Transmeta CPUs */
 386         case CPU_TRANSMETA:
 387
 388             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 389             if ((Reg[0] & 0x0FFF) >= 0x0542)
 390             {
 391                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 392                 FeatureBits |= KF_CMPXCHG8B;
 393             }
 394
 395             break;
 396
 397         /* Centaur, IDT, Rise and VIA CPUs */
 398         case CPU_CENTAUR:
 399         case CPU_RISE:
 400
 401             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 402                However, this feature exists and operates properly without any additional steps. */
 403             FeatureBits |= KF_CMPXCHG8B;
 404
 405             break;
 406     }
 407
 408     /* Convert all CPUID Feature bits into our format */
 409     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 410     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 411     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 412     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 413     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 414     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 415     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 416     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 417     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 418     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 419     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 420     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 421     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 422     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 423
 424     /* Check if the CPU has hyper-threading */
 425     if (CpuFeatures & 0x10000000)
 426     {
 427         /* Set the number of logical CPUs */
 428         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 429         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 430         {
 431             /* We're on dual-core */
 432             KiSMTProcessorsPresent = TRUE;
 433         }
 434     }
 435     else
 436     {
 437         /* We only have a single CPU */
 438         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 439     }
 440
 441     /* Check if CPUID 0x80000000 is supported */
 442     if (ExtendedCPUID)
 443     {
 444         /* Do the call */
 445         CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
 446         if ((Reg[0] & 0xffffff00) == 0x80000000)
 447         {
 448             /* Check if CPUID 0x80000001 is supported */
 449             if (Reg[0] >= 0x80000001)
 450             {
 451                 /* Check which extended features are available. */
 452                 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
 453
 454                 /* Check if NX-bit is supported */
 455                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 456
 457                 /* Now handle each features for each CPU Vendor */
 458                 switch (Vendor)
 459                 {
 460                     case CPU_AMD:
 461                     case CPU_CENTAUR:
 462                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 463                         break;
 464                 }
 465             }
 466         }
 467     }
 468
 469     /* Return the Feature Bits */
 470     return FeatureBits;
 471 }
 472
 473 VOID
 474 NTAPI
 475 KiGetCacheInformation(VOID)
 476 {
 477     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 478     ULONG Vendor;
 479     ULONG Data[4], Dummy;
 480     ULONG CacheRequests = 0, i;
 481     ULONG CurrentRegister;
 482     UCHAR RegisterByte;
 483     ULONG Size, Associativity = 0, CacheLine = 64, CurrentSize = 0;
 484     BOOLEAN FirstPass = TRUE;
 485
 486     /* Set default L2 size */
 487     Pcr->SecondLevelCacheSize = 0;
 488
 489     /* Get the Vendor ID and make sure we support CPUID */
 490     Vendor = KiGetCpuVendor();
 491     if (!Vendor) return;
 492
 493     /* Check the Vendor ID */
 494     switch (Vendor)
 495     {
 496         /* Handle Intel case */
 497         case CPU_INTEL:
 498
 499             /*Check if we support CPUID 2 */
 500             CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
 501             if (Data[0] >= 2)
 502             {
 503                 /* We need to loop for the number of times CPUID will tell us to */
 504                 do
 505                 {
 506                     /* Do the CPUID call */
 507                     CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
 508
 509                     /* Check if it was the first call */
 510                     if (FirstPass)
 511                     {
 512                         /*
 513                          * The number of times to loop is the first byte. Read
 514                          * it and then destroy it so we don't get confused.
 515                          */
 516                         CacheRequests = Data[0] & 0xFF;
 517                         Data[0] &= 0xFFFFFF00;
 518
 519                         /* Don't go over this again */
 520                         FirstPass = FALSE;
 521                     }
 522
 523                     /* Loop all 4 registers */
 524                     for (i = 0; i < 4; i++)
 525                     {
 526                         /* Get the current register */
 527                         CurrentRegister = Data[i];
 528
 529                         /*
 530                          * If the upper bit is set, then this register should
 531                          * be skipped.
 532                          */
 533                         if (CurrentRegister & 0x80000000) continue;
 534
 535                         /* Keep looping for every byte inside this register */
 536                         while (CurrentRegister)
 537                         {
 538                             /* Read a byte, skip a byte. */
 539                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 540                             CurrentRegister >>= 8;
 541                             if (!RegisterByte) continue;
 542
 543                             /*
 544                              * Valid values are from 0x40 (0 bytes) to 0x49
 545                              * (32MB), or from 0x80 to 0x89 (same size but
 546                              * 8-way associative.
 547                              */
 548                             if (((RegisterByte > 0x40) && (RegisterByte <= 0x47)) ||
 549                                 ((RegisterByte > 0x78) && (RegisterByte <= 0x7C)) ||
 550                                 ((RegisterByte > 0x80) && (RegisterByte <= 0x85)))
 551                             {
 552                                 /* Compute associativity */
 553                                 Associativity = 4;
 554                                 if (RegisterByte >= 0x79) Associativity = 8;
 555
 556                                 /* Mask out only the first nibble */
 557                                 RegisterByte &= 0x07;
 558
 559                                 /* Check if this cache is bigger than the last */
 560                                 Size = 0x10000 << RegisterByte;
 561                                 if ((Size / Associativity) > CurrentSize)
 562                                 {
 563                                     /* Set the L2 Cache Size and Associativity */
 564                                     CurrentSize = Size / Associativity;
 565                                     Pcr->SecondLevelCacheSize = Size;
 566                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 567                                 }
 568                             }
 569                             else if ((RegisterByte > 0x21) && (RegisterByte <= 0x29))
 570                             {
 571                                 /* Set minimum cache line size */
 572                                 if (CacheLine < 128) CacheLine = 128;
 573
 574                                 /* Hard-code size/associativity */
 575                                 Associativity = 8;
 576                                 switch (RegisterByte)
 577                                 {
 578                                     case 0x22:
 579                                         Size = 512 * 1024;
 580                                         Associativity = 4;
 581                                         break;
 582
 583                                     case 0x23:
 584                                         Size = 1024 * 1024;
 585                                         break;
 586
 587                                     case 0x25:
 588                                         Size = 2048 * 1024;
 589                                         break;
 590
 591                                     case 0x29:
 592                                         Size = 4096 * 1024;
 593                                         break;
 594
 595                                     default:
 596                                         Size = 0;
 597                                         break;
 598                                 }
 599
 600                                 /* Check if this cache is bigger than the last */
 601                                 if ((Size / Associativity) > CurrentSize)
 602                                 {
 603                                     /* Set the L2 Cache Size and Associativity */
 604                                     CurrentSize = Size / Associativity;
 605                                     Pcr->SecondLevelCacheSize = Size;
 606                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 607                                 }
 608                             }
 609                             else if (((RegisterByte > 0x65) && (RegisterByte < 0x69)) ||
 610                                       (RegisterByte == 0x2C) || (RegisterByte == 0xF0))
 611                             {
 612                                 /* Indicates L1 cache line of 64 bytes */
 613                                 KePrefetchNTAGranularity = 64;
 614                             }
 615                             else if (RegisterByte == 0xF1)
 616                             {
 617                                 /* Indicates L1 cache line of 128 bytes */
 618                                 KePrefetchNTAGranularity = 128;
 619                             }
 620                             else if (((RegisterByte >= 0x4A) && (RegisterByte <= 0x4C)) ||
 621                                       (RegisterByte == 0x78) ||
 622                                       (RegisterByte == 0x7D) ||
 623                                       (RegisterByte == 0x7F) ||
 624                                       (RegisterByte == 0x86) ||
 625                                       (RegisterByte == 0x87))
 626                             {
 627                                 /* Set minimum cache line size */
 628                                 if (CacheLine < 64) CacheLine = 64;
 629
 630                                 /* Hard-code size/associativity */
 631                                 switch (RegisterByte)
 632                                 {
 633                                     case 0x4A:
 634                                         Size = 4 * 1024 * 1024;
 635                                         Associativity = 8;
 636                                         break;
 637
 638                                     case 0x4B:
 639                                         Size = 6 * 1024 * 1024;
 640                                         Associativity = 12;
 641                                         break;
 642
 643                                     case 0x4C:
 644                                         Size = 8 * 1024 * 1024;
 645                                         Associativity = 16;
 646                                         break;
 647
 648                                     case 0x78:
 649                                         Size = 1 * 1024 * 1024;
 650                                         Associativity = 4;
 651                                         break;
 652
 653                                     case 0x7D:
 654                                         Size = 2 * 1024 * 1024;
 655                                         Associativity = 8;
 656                                         break;
 657
 658                                     case 0x7F:
 659                                         Size = 512 * 1024;
 660                                         Associativity = 2;
 661                                         break;
 662
 663                                     case 0x86:
 664                                         Size = 512 * 1024;
 665                                         Associativity = 4;
 666                                         break;
 667
 668                                     case 0x87:
 669                                         Size = 1 * 1024 * 1024;
 670                                         Associativity = 8;
 671                                         break;
 672
 673                                     default:
 674                                         Size = 0;
 675                                         break;
 676                                 }
 677
 678                                 /* Check if this cache is bigger than the last */
 679                                 if ((Size / Associativity) > CurrentSize)
 680                                 {
 681                                     /* Set the L2 Cache Size and Associativity */
 682                                     CurrentSize = Size / Associativity;
 683                                     Pcr->SecondLevelCacheSize = Size;
 684                                     Pcr->SecondLevelCacheAssociativity = Associativity;
 685                                 }
 686                             }
 687                         }
 688                     }
 689                 } while (--CacheRequests);
 690             }
 691             break;
 692
 693         case CPU_AMD:
 694
 695             /* Check if we support CPUID 0x80000005 */
 696             CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 697             if (Data[0] >= 0x80000006)
 698             {
 699                 /* Get L1 size first */
 700                 CPUID(0x80000005, &Data[0], &Data[1], &Data[2], &Data[3]);
 701                 KePrefetchNTAGranularity = Data[2] & 0xFF;
 702
 703                 /* Check if we support CPUID 0x80000006 */
 704                 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
 705                 if (Data[0] >= 0x80000006)
 706                 {
 707                     /* Get 2nd level cache and tlb size */
 708                     CPUID(0x80000006, &Data[0], &Data[1], &Data[2], &Data[3]);
 709
 710                     /* Cache line size */
 711                     CacheLine = Data[2] & 0xFF;
 712
 713                     /* Hardcode associativity */
 714                     RegisterByte = Data[2] >> 12;
 715                     switch (RegisterByte)
 716                     {
 717                         case 2:
 718                             Associativity = 2;
 719                             break;
 720
 721                         case 4:
 722                             Associativity = 4;
 723                             break;
 724
 725                         case 6:
 726                             Associativity = 8;
 727                             break;
 728
 729                         case 8:
 730                         case 15:
 731                             Associativity = 16;
 732                             break;
 733
 734                         default:
 735                             Associativity = 1;
 736                             break;
 737                     }
 738
 739                     /* Compute size */
 740                     Size = (Data[2] >> 16) << 10;
 741
 742                     /* Hack for Model 6, Steping 300 */
 743                     if ((KeGetCurrentPrcb()->CpuType == 6) &&
 744                         (KeGetCurrentPrcb()->CpuStep == 0x300))
 745                     {
 746                         /* Stick 64K in there */
 747                         Size = 64 * 1024;
 748                     }
 749
 750                     /* Set the L2 Cache Size and associativity */
 751                     Pcr->SecondLevelCacheSize = Size;
 752                     Pcr->SecondLevelCacheAssociativity = Associativity;
 753                 }
 754             }
 755             break;
 756
 757         case CPU_CYRIX:
 758         case CPU_TRANSMETA:
 759         case CPU_CENTAUR:
 760         case CPU_RISE:
 761
 762             /* FIXME */
 763             break;
 764     }
 765
 766     /* Set the cache line */
 767     if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
 768     DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
 769             KePrefetchNTAGranularity,
 770             Pcr->SecondLevelCacheSize,
 771             KeLargestCacheLine,
 772             Pcr->SecondLevelCacheAssociativity);
 773 }
 774
 775 VOID
 776 NTAPI
 777 KiSetCR0Bits(VOID)
 778 {
 779     ULONG Cr0;
 780
 781     /* Save current CR0 */
 782     Cr0 = __readcr0();
 783
 784     /* If this is a 486, enable Write-Protection */
 785     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 786
 787     /* Set new Cr0 */
 788     __writecr0(Cr0);
 789 }
 790
 791 VOID
 792 NTAPI
 793 KiInitializeTSS2(IN PKTSS Tss,
 794                  IN PKGDTENTRY TssEntry OPTIONAL)
 795 {
 796     PUCHAR p;
 797
 798     /* Make sure the GDT Entry is valid */
 799     if (TssEntry)
 800     {
 801         /* Set the Limit */
 802         TssEntry->LimitLow = sizeof(KTSS) - 1;
 803         TssEntry->HighWord.Bits.LimitHi = 0;
 804     }
 805
 806     /* Now clear the I/O Map */
 807     ASSERT(IOPM_COUNT == 1);
 808     RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
 809
 810     /* Initialize Interrupt Direction Maps */
 811     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 812     RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
 813
 814     /* Add DPMI support for interrupts */
 815     p[0] = 4;
 816     p[3] = 0x18;
 817     p[4] = 0x18;
 818
 819     /* Initialize the default Interrupt Direction Map */
 820     p = Tss->IntDirectionMap;
 821     RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
 822
 823     /* Add DPMI support */
 824     p[0] = 4;
 825     p[3] = 0x18;
 826     p[4] = 0x18;
 827 }
 828
 829 VOID
 830 NTAPI
 831 KiInitializeTSS(IN PKTSS Tss)
 832 {
 833     /* Set an invalid map base */
 834     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 835
 836     /* Disable traps during Task Switches */
 837     Tss->Flags = 0;
 838
 839     /* Set LDT and Ring 0 SS */
 840     Tss->LDT = 0;
 841     Tss->Ss0 = KGDT_R0_DATA;
 842 }
 843
 844 VOID
 845 FASTCALL
 846 Ki386InitializeTss(IN PKTSS Tss,
 847                    IN PKIDTENTRY Idt,
 848                    IN PKGDTENTRY Gdt)
 849 {
 850     PKGDTENTRY TssEntry, TaskGateEntry;
 851
 852     /* Initialize the boot TSS. */
 853     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 854     TssEntry->HighWord.Bits.Type = I386_TSS;
 855     TssEntry->HighWord.Bits.Pres = 1;
 856     TssEntry->HighWord.Bits.Dpl = 0;
 857     KiInitializeTSS2(Tss, TssEntry);
 858     KiInitializeTSS(Tss);
 859
 860     /* Load the task register */
 861     Ke386SetTr(KGDT_TSS);
 862
 863     /* Setup the Task Gate for Double Fault Traps */
 864     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 865     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 866     TaskGateEntry->HighWord.Bits.Pres = 1;
 867     TaskGateEntry->HighWord.Bits.Dpl = 0;
 868     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 869
 870     /* Initialize the TSS used for handling double faults. */
 871     Tss = (PKTSS)KiDoubleFaultTSS;
 872     KiInitializeTSS(Tss);
 873     Tss->CR3 = __readcr3();
 874     Tss->Esp0 = KiDoubleFaultStack;
 875     Tss->Esp = KiDoubleFaultStack;
 876     Tss->Eip = PtrToUlong(KiTrap08);
 877     Tss->Cs = KGDT_R0_CODE;
 878     Tss->Fs = KGDT_R0_PCR;
 879     Tss->Ss = Ke386GetSs();
 880     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 881     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 882
 883     /* Setup the Double Trap TSS entry in the GDT */
 884     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 885     TssEntry->HighWord.Bits.Type = I386_TSS;
 886     TssEntry->HighWord.Bits.Pres = 1;
 887     TssEntry->HighWord.Bits.Dpl = 0;
 888     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 889     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 890     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 891     TssEntry->LimitLow = KTSS_IO_MAPS;
 892
 893     /* Now setup the NMI Task Gate */
 894     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 895     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 896     TaskGateEntry->HighWord.Bits.Pres = 1;
 897     TaskGateEntry->HighWord.Bits.Dpl = 0;
 898     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 899
 900     /* Initialize the actual TSS */
 901     Tss = (PKTSS)KiNMITSS;
 902     KiInitializeTSS(Tss);
 903     Tss->CR3 = __readcr3();
 904     Tss->Esp0 = KiDoubleFaultStack;
 905     Tss->Esp = KiDoubleFaultStack;
 906     Tss->Eip = PtrToUlong(KiTrap02);
 907     Tss->Cs = KGDT_R0_CODE;
 908     Tss->Fs = KGDT_R0_PCR;
 909     Tss->Ss = Ke386GetSs();
 910     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 911     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 912
 913     /* And its associated TSS Entry */
 914     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 915     TssEntry->HighWord.Bits.Type = I386_TSS;
 916     TssEntry->HighWord.Bits.Pres = 1;
 917     TssEntry->HighWord.Bits.Dpl = 0;
 918     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 919     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 920     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 921     TssEntry->LimitLow = KTSS_IO_MAPS;
 922 }
 923
 924 VOID
 925 NTAPI
 926 KeFlushCurrentTb(VOID)
 927 {
 928     /* Flush the TLB by resetting CR3 */
 929     __writecr3(__readcr3());
 930 }
 931
 932 VOID
 933 NTAPI
 934 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 935 {
 936     PKGDTENTRY TssEntry;
 937
 938     //
 939     // Restore the CR registers
 940     //
 941     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 942     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 943     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 944     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 945
 946     //
 947     // Restore the DR registers
 948     //
 949     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 950     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 951     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 952     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 953     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 954     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 955
 956     //
 957     // Restore GDT and IDT
 958     //
 959     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 960     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 961
 962     //
 963     // Clear the busy flag so we don't crash if we reload the same selector
 964     //
 965     TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
 966                             ProcessorState->SpecialRegisters.Tr);
 967     TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
 968
 969     //
 970     // Restore TSS and LDT
 971     //
 972     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 973     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 974 }
 975
 976 VOID
 977 NTAPI
 978 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 979 {
 980     /* Save the CR registers */
 981     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 982     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 983     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 984     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 985                                            __readcr4() : 0;
 986
 987     /* Save the DR registers */
 988     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 989     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 990     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 991     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 992     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 993     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 994     __writedr(7, 0);
 995
 996     /* Save GDT, IDT, LDT and TSS */
 997     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 998     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 999     ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
1000     ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
1001 }
1002
1003 VOID
1004 NTAPI
1005 KiInitializeMachineType(VOID)
1006 {
1007     /* Set the Machine Type we got from NTLDR */
1008     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
1009 }
1010
1011 ULONG_PTR
1012 NTAPI
1013 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
1014 {
1015     /* Set CS and ESP */
1016     WRMSR(0x174, KGDT_R0_CODE);
1017     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
1018
1019     /* Set LSTAR */
1020     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
1021     return 0;
1022 }
1023
1024 VOID
1025 NTAPI
1026 KiRestoreFastSyscallReturnState(VOID)
1027 {
1028     /* Check if the CPU Supports fast system call */
1029     if (KeFeatureBits & KF_FAST_SYSCALL)
1030     {
1031         /* Check if it has been disabled */
1032         if (!KiFastSystemCallDisable)
1033         {
1034             /* Do an IPI to enable it */
1035             KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
1036
1037             /* It's enabled, so use the proper exit stub */
1038             KiFastCallExitHandler = KiSystemCallSysExitReturn;
1039             DPRINT1("Support for SYSENTER detected.\n");
1040         }
1041         else
1042         {
1043             /* Disable fast system call */
1044             KeFeatureBits &= ~KF_FAST_SYSCALL;
1045             KiFastCallExitHandler = KiSystemCallTrapReturn;
1046             DPRINT1("Support for SYSENTER disabled.\n");
1047         }
1048     }
1049     else
1050     {
1051         /* Use the IRET handler */
1052         KiFastCallExitHandler = KiSystemCallTrapReturn;
1053         DPRINT1("No support for SYSENTER detected.\n");
1054     }
1055 }
1056
1057 ULONG_PTR
1058 NTAPI
1059 Ki386EnableDE(IN ULONG_PTR Context)
1060 {
1061     /* Enable DE */
1062     __writecr4(__readcr4() | CR4_DE);
1063     return 0;
1064 }
1065
1066 ULONG_PTR
1067 NTAPI
1068 Ki386EnableFxsr(IN ULONG_PTR Context)
1069 {
1070     /* Enable FXSR */
1071     __writecr4(__readcr4() | CR4_FXSR);
1072     return 0;
1073 }
1074
1075 ULONG_PTR
1076 NTAPI
1077 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
1078 {
1079     PKIDTENTRY IdtEntry;
1080
1081     /* Get the IDT Entry for Interrupt 0x13 */
1082     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
1083
1084     /* Set it up */
1085     IdtEntry->Selector = KGDT_R0_CODE;
1086     IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
1087     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
1088     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
1089     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
1090     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
1091
1092     /* Enable XMMI exceptions */
1093     __writecr4(__readcr4() | CR4_XMMEXCPT);
1094     return 0;
1095 }
1096
1097 VOID
1098 NTAPI
1099 KiI386PentiumLockErrataFixup(VOID)
1100 {
1101     KDESCRIPTOR IdtDescriptor;
1102     PKIDTENTRY NewIdt, NewIdt2;
1103
1104     /* Allocate memory for a new IDT */
1105     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
1106
1107     /* Put everything after the first 7 entries on a new page */
1108     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
1109
1110     /* Disable interrupts */
1111     _disable();
1112
1113     /* Get the current IDT and copy it */
1114     __sidt(&IdtDescriptor.Limit);
1115     RtlCopyMemory(NewIdt2,
1116                   (PVOID)IdtDescriptor.Base,
1117                   IdtDescriptor.Limit + 1);
1118     IdtDescriptor.Base = (ULONG)NewIdt2;
1119
1120     /* Set the new IDT */
1121     __lidt(&IdtDescriptor.Limit);
1122     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
1123
1124     /* Restore interrupts */
1125     _enable();
1126
1127     /* Set the first 7 entries as read-only to produce a fault */
1128     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
1129 }
1130
1131 BOOLEAN
1132 NTAPI
1133 KeDisableInterrupts(VOID)
1134 {
1135     ULONG Flags;
1136     BOOLEAN Return;
1137
1138     /* Get EFLAGS and check if the interrupt bit is set */
1139     Flags = __readeflags();
1140     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
1141
1142     /* Disable interrupts */
1143     _disable();
1144     return Return;
1145 }
1146
1147 BOOLEAN
1148 NTAPI
1149 KeInvalidateAllCaches(VOID)
1150 {
1151     /* Only supported on Pentium Pro and higher */
1152     if (KeI386CpuType < 6) return FALSE;
1153
1154     /* Invalidate all caches */
1155     __wbinvd();
1156     return TRUE;
1157 }
1158
1159 VOID
1160 FASTCALL
1161 KeZeroPages(IN PVOID Address,
1162             IN ULONG Size)
1163 {
1164     /* Not using XMMI in this routine */
1165     RtlZeroMemory(Address, Size);
1166 }
1167
1168 VOID
1169 NTAPI
1170 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
1171                      IN PKEXCEPTION_FRAME ExceptionFrame)
1172 {
1173     PKPRCB Prcb = KeGetCurrentPrcb();
1174
1175     //
1176     // Save full context
1177     //
1178     Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
1179                                                      CONTEXT_DEBUG_REGISTERS;
1180     KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
1181
1182     //
1183     // Save control registers
1184     //
1185     KiSaveProcessorControlState(&Prcb->ProcessorState);
1186 }
1187
1188 BOOLEAN
1189 NTAPI
1190 KiIsNpxPresent(VOID)
1191 {
1192     ULONG Cr0;
1193     USHORT Magic;
1194
1195     /* Set magic */
1196     Magic = 0xFFFF;
1197
1198     /* Read CR0 and mask out FPU flags */
1199     Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
1200
1201     /* Store on FPU stack */
1202 #ifdef _MSC_VER
1203     __asm fninit;
1204     __asm fnstsw Magic;
1205 #else
1206     asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
1207 #endif
1208
1209     /* Magic should now be cleared */
1210     if (Magic & 0xFF)
1211     {
1212         /* You don't have an FPU -- enable emulation for now */
1213         __writecr0(Cr0 | CR0_EM | CR0_TS);
1214         return FALSE;
1215     }
1216
1217     /* You have an FPU, enable it */
1218     Cr0 |= CR0_ET;
1219
1220     /* Enable INT 16 on 486 and higher */
1221     if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
1222
1223     /* Set FPU state */
1224     __writecr0(Cr0 | CR0_EM | CR0_TS);
1225     return TRUE;
1226 }
1227
1228 BOOLEAN
1229 NTAPI
1230 KiIsNpxErrataPresent(VOID)
1231 {
1232     BOOLEAN ErrataPresent;
1233     ULONG Cr0;
1234     volatile double Value1, Value2;
1235
1236     /* Disable interrupts */
1237     _disable();
1238
1239     /* Read CR0 and remove FPU flags */
1240     Cr0 = __readcr0();
1241     __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1242
1243     /* Initialize FPU state */
1244     Ke386FnInit();
1245
1246     /* Multiply the magic values and divide, we should get the result back */
1247     Value1 = 4195835.0;
1248     Value2 = 3145727.0;
1249     ErrataPresent = (Value1 * Value2 / 3145727.0) != 4195835.0;
1250
1251     /* Restore CR0 */
1252     __writecr0(Cr0);
1253
1254     /* Enable interrupts */
1255     _enable();
1256
1257     /* Return if there's an errata */
1258     return ErrataPresent;
1259 }
1260
1261 VOID
1262 NTAPI
1263 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
1264 {
1265     ULONG EFlags, Cr0;
1266     PKTHREAD Thread, NpxThread;
1267     PFX_SAVE_AREA FxSaveArea;
1268
1269     /* Save volatiles and disable interrupts */
1270     EFlags = __readeflags();
1271     _disable();
1272
1273     /* Save the PCR and get the current thread */
1274     Thread = KeGetCurrentThread();
1275
1276     /* Check if we're already loaded */
1277     if (Thread->NpxState != NPX_STATE_LOADED)
1278     {
1279         /* If there's nothing to load, quit */
1280         if (!SaveArea) return;
1281
1282         /* Need FXSR support for this */
1283         ASSERT(KeI386FxsrPresent == TRUE);
1284
1285         /* Check for sane CR0 */
1286         Cr0 = __readcr0();
1287         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1288         {
1289             /* Mask out FPU flags */
1290             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1291         }
1292
1293         /* Get the NPX thread and check its FPU state */
1294         NpxThread = KeGetCurrentPrcb()->NpxThread;
1295         if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
1296         {
1297             /* Get the FX frame and store the state there */
1298             FxSaveArea = KiGetThreadNpxArea(NpxThread);
1299             Ke386FxSave(FxSaveArea);
1300
1301             /* NPX thread has lost its state */
1302             NpxThread->NpxState = NPX_STATE_NOT_LOADED;
1303         }
1304
1305         /* Now load NPX state from the NPX area */
1306         FxSaveArea = KiGetThreadNpxArea(Thread);
1307         Ke386FxStore(FxSaveArea);
1308     }
1309     else
1310     {
1311         /* Check for sane CR0 */
1312         Cr0 = __readcr0();
1313         if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1314         {
1315             /* Mask out FPU flags */
1316             __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1317         }
1318
1319         /* Get FX frame */
1320         FxSaveArea = KiGetThreadNpxArea(Thread);
1321         Thread->NpxState = NPX_STATE_NOT_LOADED;
1322
1323         /* Save state if supported by CPU */
1324         if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
1325     }
1326
1327     /* Now save the FN state wherever it was requested */
1328     if (SaveArea) Ke386FnSave(SaveArea);
1329
1330     /* Clear NPX thread */
1331     KeGetCurrentPrcb()->NpxThread = NULL;
1332
1333     /* Add the CR0 from the NPX frame */
1334     Cr0 |= NPX_STATE_NOT_LOADED;
1335     Cr0 |= FxSaveArea->Cr0NpxState;
1336     __writecr0(Cr0);
1337
1338     /* Restore interrupt state */
1339     __writeeflags(EFlags);
1340 }
1341
1342 /* PUBLIC FUNCTIONS **********************************************************/
1343
1344 /*
1345  * @implemented
1346  */
1347 VOID
1348 NTAPI
1349 KiCoprocessorError(VOID)
1350 {
1351     PFX_SAVE_AREA NpxArea;
1352
1353     /* Get the FPU area */
1354     NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
1355
1356     /* Set CR0_TS */
1357     NpxArea->Cr0NpxState = CR0_TS;
1358     __writecr0(__readcr0() | CR0_TS);
1359 }
1360
1361 /*
1362  * @implemented
1363  */
1364 NTSTATUS
1365 NTAPI
1366 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
1367 {
1368     PFNSAVE_FORMAT FpState;
1369     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
1370     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1371
1372     /* check if we are doing software emulation */
1373     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
1374
1375     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1376     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1377
1378     *((PVOID *) Save) = FpState;
1379 #ifdef __GNUC__
1380     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1381 #else
1382     __asm
1383     {
1384         fnsave [FpState]
1385     };
1386 #endif
1387
1388     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1389     return STATUS_SUCCESS;
1390 }
1391
1392 /*
1393  * @implemented
1394  */
1395 NTSTATUS
1396 NTAPI
1397 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1398 {
1399     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1400     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1401     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1402
1403 #ifdef __GNUC__
1404     asm volatile("fnclex\n\t");
1405     asm volatile("frstor %0\n\t" : "=m" (*FpState));
1406 #else
1407     __asm
1408     {
1409         fnclex
1410         frstor [FpState]
1411     };
1412 #endif
1413
1414     ExFreePool(FpState);
1415     return STATUS_SUCCESS;
1416 }
1417
1418 /*
1419  * @implemented
1420  */
1421 ULONG
1422 NTAPI
1423 KeGetRecommendedSharedDataAlignment(VOID)
1424 {
1425     /* Return the global variable */
1426     return KeLargestCacheLine;
1427 }
1428
1429 VOID
1430 NTAPI
1431 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1432                       IN PVOID Ignored1,
1433                       IN PVOID Ignored2,
1434                       IN PVOID Ignored3)
1435 {
1436     /* Signal this packet as done */
1437     KiIpiSignalPacketDone(PacketContext);
1438
1439     /* Flush the TB for the Current CPU */
1440     KeFlushCurrentTb();
1441 }
1442
1443 /*
1444  * @implemented
1445  */
1446 VOID
1447 NTAPI
1448 KeFlushEntireTb(IN BOOLEAN Invalid,
1449                 IN BOOLEAN AllProcessors)
1450 {
1451     KIRQL OldIrql;
1452 #ifdef CONFIG_SMP
1453     KAFFINITY TargetAffinity;
1454     PKPRCB Prcb = KeGetCurrentPrcb();
1455 #endif
1456
1457     /* Raise the IRQL for the TB Flush */
1458     OldIrql = KeRaiseIrqlToSynchLevel();
1459
1460 #ifdef CONFIG_SMP
1461     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1462
1463     /* Get the current processor affinity, and exclude ourselves */
1464     TargetAffinity = KeActiveProcessors;
1465     TargetAffinity &= ~Prcb->SetMember;
1466
1467     /* Make sure this is MP */
1468     if (TargetAffinity)
1469     {
1470         /* Send an IPI TB flush to the other processors */
1471         KiIpiSendPacket(TargetAffinity,
1472                         KiFlushTargetEntireTb,
1473                         NULL,
1474                         0,
1475                         NULL);
1476     }
1477 #endif
1478
1479     /* Flush the TB for the Current CPU, and update the flush stamp */
1480     KeFlushCurrentTb();
1481
1482 #ifdef CONFIG_SMP
1483     /* If this is MP, wait for the other processors to finish */
1484     if (TargetAffinity)
1485     {
1486         /* Sanity check */
1487         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1488
1489         /* FIXME: TODO */
1490         ASSERTMSG("Not yet implemented\n", FALSE);
1491     }
1492 #endif
1493
1494     /* Update the flush stamp and return to original IRQL */
1495     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1496     KeLowerIrql(OldIrql);
1497 }
1498
1499 /*
1500  * @implemented
1501  */
1502 VOID
1503 NTAPI
1504 KeSetDmaIoCoherency(IN ULONG Coherency)
1505 {
1506     /* Save the coherency globally */
1507     KiDmaIoCoherency = Coherency;
1508 }
1509
1510 /*
1511  * @implemented
1512  */
1513 KAFFINITY
1514 NTAPI
1515 KeQueryActiveProcessors(VOID)
1516 {
1517     PAGED_CODE();
1518
1519     /* Simply return the number of active processors */
1520     return KeActiveProcessors;
1521 }
1522
1523 /*
1524  * @implemented
1525  */
1526 VOID
1527 __cdecl
1528 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1529 {
1530     /* Capture the context */
1531     RtlCaptureContext(&State->ContextFrame);
1532
1533     /* Capture the control state */
1534     KiSaveProcessorControlState(State);
1535 }