ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* FIXME: Local EFLAGS defines not used anywhere else */
  16 #define EFLAGS_IOPL     0x3000
  17 #define EFLAGS_NF       0x4000
  18 #define EFLAGS_RF       0x10000
  19 #define EFLAGS_ID       0x200000
  20
  21 /* GLOBALS *******************************************************************/
  22
  23 /* The Boot TSS */
  24 KTSS KiBootTss;
  25
  26 /* The TSS to use for Double Fault Traps (INT 0x9) */
  27 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  28
  29 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  30 UCHAR KiNMITSS[KTSS_IO_MAPS];
  31
  32 /* The Boot GDT */
  33 KGDTENTRY KiBootGdt[256] =
  34 {
  35     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_NULL */
  36     {0xffff, 0x0000, {{0x00, 0x9b, 0xcf, 0x00}}},       /* KGDT_R0_CODE */
  37     {0xffff, 0x0000, {{0x00, 0x93, 0xcf, 0x00}}},       /* KGDT_R0_DATA */
  38     {0xffff, 0x0000, {{0x00, 0xfb, 0xcf, 0x00}}},       /* KGDT_R3_CODE */
  39     {0xffff, 0x0000, {{0x00, 0xf3, 0xcf, 0x00}}},       /* KGDT_R3_DATA*/
  40     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_TSS */
  41     {0x0001, 0xf000, {{0xdf, 0x93, 0xc0, 0xff}}},       /* KGDT_R0_PCR */
  42     {0x0fff, 0x0000, {{0x00, 0xf3, 0x40, 0x00}}},       /* KGDT_R3_TEB */
  43     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_UNUSED */
  44     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_LDT */
  45     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_DF_TSS */
  46     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}        /* KGDT_NMI_TSS */
  47 };
  48
  49 /* GDT Descriptor */
  50 KDESCRIPTOR KiGdtDescriptor = {0, sizeof(KiBootGdt) - 1, (ULONG)KiBootGdt};
  51
  52 /* CPU Features and Flags */
  53 ULONG KeI386CpuType;
  54 ULONG KeI386CpuStep;
  55 ULONG KeProcessorArchitecture;
  56 ULONG KeProcessorLevel;
  57 ULONG KeProcessorRevision;
  58 ULONG KeFeatureBits;
  59 ULONG KiFastSystemCallDisable = 1;
  60 ULONG KeI386NpxPresent = 0;
  61 ULONG KiMXCsrMask = 0;
  62 ULONG MxcsrFeatureMask = 0;
  63 ULONG KeI386XMMIPresent = 0;
  64 ULONG KeI386FxsrPresent = 0;
  65 ULONG KeI386MachineType;
  66 ULONG Ke386Pae = FALSE;
  67 ULONG Ke386NoExecute = FALSE;
  68 ULONG KeLargestCacheLine = 0x40;
  69 ULONG KeDcacheFlushCount = 0;
  70 ULONG KeIcacheFlushCount = 0;
  71 ULONG KiDmaIoCoherency = 0;
  72 CHAR KeNumberProcessors;
  73 KAFFINITY KeActiveProcessors = 1;
  74 BOOLEAN KiI386PentiumLockErrataPresent;
  75 BOOLEAN KiSMTProcessorsPresent;
  76
  77 /* Freeze data */
  78 KIRQL KiOldIrql;
  79 ULONG KiFreezeFlag;
  80
  81 /* Flush data */
  82 volatile LONG KiTbFlushTimeStamp;
  83
  84 /* CPU Signatures */
  85 static const CHAR CmpIntelID[]       = "GenuineIntel";
  86 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  87 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  88 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  89 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  90 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  91
  92 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  93
  94 VOID
  95 NTAPI
  96 CPUID(OUT ULONG CpuInfo[4],
  97       IN ULONG InfoType)
  98 {
  99     Ki386Cpuid(InfoType, &CpuInfo[0], &CpuInfo[1], &CpuInfo[2], &CpuInfo[3]);
 100 }
 101
 102 VOID
 103 WRMSR(IN ULONG Register,
 104       IN LONGLONG Value)
 105 {
 106     LARGE_INTEGER LargeVal;
 107     LargeVal.QuadPart = Value;
 108     Ke386Wrmsr(Register, LargeVal.HighPart, LargeVal.LowPart);
 109 }
 110
 111 LONGLONG
 112 RDMSR(IN ULONG Register)
 113 {
 114     LARGE_INTEGER LargeVal = {{0}};
 115     Ke386Rdmsr(Register, LargeVal.HighPart, LargeVal.LowPart);
 116     return LargeVal.QuadPart;
 117 }
 118
 119 /* FUNCTIONS *****************************************************************/
 120
 121 VOID
 122 NTAPI
 123 KiSetProcessorType(VOID)
 124 {
 125     ULONG EFlags = 0, NewEFlags;
 126     ULONG Reg[4];
 127     ULONG Stepping, Type;
 128
 129     /* Start by assuming no CPUID data */
 130     KeGetCurrentPrcb()->CpuID = 0;
 131
 132     /* Save EFlags */
 133     Ke386SaveFlags(EFlags);
 134
 135     /* XOR out the ID bit and update EFlags */
 136     NewEFlags = EFlags ^ EFLAGS_ID;
 137     Ke386RestoreFlags(NewEFlags);
 138
 139     /* Get them back and see if they were modified */
 140     Ke386SaveFlags(NewEFlags);
 141     if (NewEFlags != EFlags)
 142     {
 143         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 144         EFlags |= EFLAGS_ID;
 145         Ke386RestoreFlags(EFlags);
 146
 147         /* Peform CPUID 0 to see if CPUID 1 is supported */
 148         CPUID(Reg, 0);
 149         if (Reg[0] > 0)
 150         {
 151             /* Do CPUID 1 now */
 152             CPUID(Reg, 1);
 153
 154             /*
 155              * Get the Stepping and Type. The stepping contains both the
 156              * Model and the Step, while the Type contains the returned Type.
 157              * We ignore the family.
 158              *
 159              * For the stepping, we convert this: zzzzzzxy into this: x0y
 160              */
 161             Stepping = Reg[0] & 0xF0;
 162             Stepping <<= 4;
 163             Stepping += (Reg[0] & 0xFF);
 164             Stepping &= 0xF0F;
 165             Type = Reg[0] & 0xF00;
 166             Type >>= 8;
 167
 168             /* Save them in the PRCB */
 169             KeGetCurrentPrcb()->CpuID = TRUE;
 170             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 171             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 172         }
 173         else
 174         {
 175             DPRINT1("CPUID Support lacking\n");
 176         }
 177     }
 178     else
 179     {
 180         DPRINT1("CPUID Support lacking\n");
 181     }
 182
 183     /* Restore EFLAGS */
 184     Ke386RestoreFlags(EFlags);
 185 }
 186
 187 ULONG
 188 NTAPI
 189 KiGetCpuVendor(VOID)
 190 {
 191     PKPRCB Prcb = KeGetCurrentPrcb();
 192     ULONG Vendor[5];
 193     ULONG Temp;
 194
 195     /* Assume no Vendor ID and fail if no CPUID Support. */
 196     Prcb->VendorString[0] = 0;
 197     if (!Prcb->CpuID) return 0;
 198
 199     /* Get the Vendor ID and null-terminate it */
 200     CPUID(Vendor, 0);
 201     Vendor[4] = 0;
 202
 203     /* Re-arrange vendor string */
 204     Temp = Vendor[2];
 205     Vendor[2] = Vendor[3];
 206     Vendor[3] = Temp;
 207
 208     /* Copy it to the PRCB and null-terminate it again */
 209     RtlCopyMemory(Prcb->VendorString,
 210                   &Vendor[1],
 211                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 212     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 213
 214     /* Now check the CPU Type */
 215     if (!strcmp(Prcb->VendorString, CmpIntelID))
 216     {
 217         return CPU_INTEL;
 218     }
 219     else if (!strcmp(Prcb->VendorString, CmpAmdID))
 220     {
 221         return CPU_AMD;
 222     }
 223     else if (!strcmp(Prcb->VendorString, CmpCyrixID))
 224     {
 225         DPRINT1("Cyrix CPUs not fully supported\n");
 226         return 0;
 227     }
 228     else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
 229     {
 230         DPRINT1("Transmeta CPUs not fully supported\n");
 231         return 0;
 232     }
 233     else if (!strcmp(Prcb->VendorString, CmpCentaurID))
 234     {
 235         DPRINT1("VIA CPUs not fully supported\n");
 236         return 0;
 237     }
 238     else if (!strcmp(Prcb->VendorString, CmpRiseID))
 239     {
 240         DPRINT1("Rise CPUs not fully supported\n");
 241         return 0;
 242     }
 243
 244     /* Invalid CPU */
 245     return 0;
 246 }
 247
 248 ULONG
 249 NTAPI
 250 KiGetFeatureBits(VOID)
 251 {
 252     PKPRCB Prcb = KeGetCurrentPrcb();
 253     ULONG Vendor;
 254     ULONG FeatureBits = KF_WORKING_PTE;
 255     ULONG Reg[4];
 256     BOOLEAN ExtendedCPUID = TRUE;
 257     ULONG CpuFeatures = 0;
 258
 259     /* Get the Vendor ID */
 260     Vendor = KiGetCpuVendor();
 261
 262     /* Make sure we got a valid vendor ID at least. */
 263     if (!Vendor) return FeatureBits;
 264
 265     /* Get the CPUID Info. Features are in Reg[3]. */
 266     CPUID(Reg, 1);
 267
 268     /* Set the initial APIC ID */
 269     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 270
 271     /* Check for AMD CPU */
 272     if (Vendor == CPU_AMD)
 273     {
 274         /* Check if this is a K5 or higher. */
 275         if ((Reg[0] & 0x0F00) >= 0x0500)
 276         {
 277             /* Check if this is a K5 specifically. */
 278             if ((Reg[0] & 0x0F00) == 0x0500)
 279             {
 280                 /* Get the Model Number */
 281                 switch (Reg[0] & 0x00F0)
 282                 {
 283                     /* Check if this is the Model 1 */
 284                     case 0x0010:
 285
 286                         /* Check if this is Step 0 or 1. They don't support PGE */
 287                         if ((Reg[0] & 0x000F) > 0x03) break;
 288
 289                     case 0x0000:
 290
 291                         /* Model 0 doesn't support PGE at all. */
 292                         Reg[3] &= ~0x2000;
 293                         break;
 294
 295                     case 0x0080:
 296
 297                         /* K6-2, Step 8 and over have support for MTRR. */
 298                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 299                         break;
 300
 301                     case 0x0090:
 302
 303                         /* As does the K6-3 */
 304                         FeatureBits |= KF_AMDK6MTRR;
 305                         break;
 306
 307                     default:
 308                         break;
 309                 }
 310             }
 311         }
 312         else
 313         {
 314             /* Families below 5 don't support PGE, PSE or CMOV at all */
 315             Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 316
 317             /* They also don't support advanced CPUID functions. */
 318             ExtendedCPUID = FALSE;
 319         }
 320
 321         /* Set the current features */
 322         CpuFeatures = Reg[3];
 323     }
 324
 325     /* Now check if this is Intel */
 326     if (Vendor == CPU_INTEL)
 327     {
 328         /* Check if it's a P6 */
 329         if (Prcb->CpuType == 6)
 330         {
 331             /* Perform the special sequence to get the MicroCode Signature */
 332             WRMSR(0x8B, 0);
 333             CPUID(Reg, 1);
 334             Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 335         }
 336         else if (Prcb->CpuType == 5)
 337         {
 338             /* On P5, enable workaround for the LOCK errata. */
 339             KiI386PentiumLockErrataPresent = TRUE;
 340         }
 341
 342         /* Check for broken P6 with bad SMP PTE implementation */
 343         if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 344             ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 345         {
 346             /* Remove support for correct PTE support. */
 347             FeatureBits &= ~KF_WORKING_PTE;
 348         }
 349
 350         /* Check if the CPU is too old to support SYSENTER */
 351         if ((Prcb->CpuType < 6) ||
 352             ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 353         {
 354             /* Disable it */
 355             Reg[3] &= ~0x800;
 356         }
 357
 358         /* Set the current features */
 359         CpuFeatures = Reg[3];
 360     }
 361
 362     /* Convert all CPUID Feature bits into our format */
 363     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 364     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 365     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 366     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 367     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 368     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 369     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 370     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 371     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 372     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 373     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 374     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 375     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 376     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 377
 378     /* Check if the CPU has hyper-threading */
 379     if (CpuFeatures & 0x10000000)
 380     {
 381         /* Set the number of logical CPUs */
 382         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 383         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 384         {
 385             /* We're on dual-core */
 386             KiSMTProcessorsPresent = TRUE;
 387         }
 388     }
 389     else
 390     {
 391         /* We only have a single CPU */
 392         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 393     }
 394
 395     /* Check if CPUID 0x80000000 is supported */
 396     if (ExtendedCPUID)
 397     {
 398         /* Do the call */
 399         CPUID(Reg, 0x80000000);
 400         if ((Reg[0] & 0xffffff00) == 0x80000000)
 401         {
 402             /* Check if CPUID 0x80000001 is supported */
 403             if (Reg[0] >= 0x80000001)
 404             {
 405                 /* Check which extended features are available. */
 406                 CPUID(Reg, 0x80000001);
 407
 408                 /* Check if NX-bit is supported */
 409                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 410
 411                 /* Now handle each features for each CPU Vendor */
 412                 switch (Vendor)
 413                 {
 414                     case CPU_AMD:
 415                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 416                         break;
 417                 }
 418             }
 419         }
 420     }
 421
 422     /* Return the Feature Bits */
 423     return FeatureBits;
 424 }
 425
 426 VOID
 427 NTAPI
 428 KiGetCacheInformation(VOID)
 429 {
 430     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 431     ULONG Vendor;
 432     ULONG Data[4];
 433     ULONG CacheRequests = 0, i;
 434     ULONG CurrentRegister;
 435     UCHAR RegisterByte;
 436     BOOLEAN FirstPass = TRUE;
 437
 438     /* Set default L2 size */
 439     Pcr->SecondLevelCacheSize = 0;
 440
 441     /* Get the Vendor ID and make sure we support CPUID */
 442     Vendor = KiGetCpuVendor();
 443     if (!Vendor) return;
 444
 445     /* Check the Vendor ID */
 446     switch (Vendor)
 447     {
 448         /* Handle Intel case */
 449         case CPU_INTEL:
 450
 451             /*Check if we support CPUID 2 */
 452             CPUID(Data, 0);
 453             if (Data[0] >= 2)
 454             {
 455                 /* We need to loop for the number of times CPUID will tell us to */
 456                 do
 457                 {
 458                     /* Do the CPUID call */
 459                     CPUID(Data, 2);
 460
 461                     /* Check if it was the first call */
 462                     if (FirstPass)
 463                     {
 464                         /*
 465                          * The number of times to loop is the first byte. Read
 466                          * it and then destroy it so we don't get confused.
 467                          */
 468                         CacheRequests = Data[0] & 0xFF;
 469                         Data[0] &= 0xFFFFFF00;
 470
 471                         /* Don't go over this again */
 472                         FirstPass = FALSE;
 473                     }
 474
 475                     /* Loop all 4 registers */
 476                     for (i = 0; i < 4; i++)
 477                     {
 478                         /* Get the current register */
 479                         CurrentRegister = Data[i];
 480
 481                         /*
 482                          * If the upper bit is set, then this register should
 483                          * be skipped.
 484                          */
 485                         if (CurrentRegister & 0x80000000) continue;
 486
 487                         /* Keep looping for every byte inside this register */
 488                         while (CurrentRegister)
 489                         {
 490                             /* Read a byte, skip a byte. */
 491                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 492                             CurrentRegister >>= 8;
 493                             if (!RegisterByte) continue;
 494
 495                             /*
 496                              * Valid values are from 0x40 (0 bytes) to 0x49
 497                              * (32MB), or from 0x80 to 0x89 (same size but
 498                              * 8-way associative.
 499                              */
 500                             if (((RegisterByte > 0x40) &&
 501                                  (RegisterByte <= 0x49)) ||
 502                                 ((RegisterByte > 0x80) &&
 503                                 (RegisterByte <= 0x89)))
 504                             {
 505                                 /* Mask out only the first nibble */
 506                                 RegisterByte &= 0x0F;
 507
 508                                 /* Set the L2 Cache Size */
 509                                 Pcr->SecondLevelCacheSize = 0x10000 <<
 510                                                             RegisterByte;
 511                             }
 512                         }
 513                     }
 514                 } while (--CacheRequests);
 515             }
 516             break;
 517
 518         case CPU_AMD:
 519
 520             /* Check if we support CPUID 0x80000006 */
 521             CPUID(Data, 0x80000000);
 522             if (Data[0] >= 6)
 523             {
 524                 /* Get 2nd level cache and tlb size */
 525                 CPUID(Data, 0x80000006);
 526
 527                 /* Set the L2 Cache Size */
 528                 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
 529             }
 530             break;
 531     }
 532 }
 533
 534 VOID
 535 NTAPI
 536 KiSetCR0Bits(VOID)
 537 {
 538     ULONG Cr0;
 539
 540     /* Save current CR0 */
 541     Cr0 = __readcr0();
 542
 543     /* If this is a 486, enable Write-Protection */
 544     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 545
 546     /* Set new Cr0 */
 547     __writecr0(Cr0);
 548 }
 549
 550 VOID
 551 NTAPI
 552 KiInitializeTSS2(IN PKTSS Tss,
 553                  IN PKGDTENTRY TssEntry OPTIONAL)
 554 {
 555     PUCHAR p;
 556
 557     /* Make sure the GDT Entry is valid */
 558     if (TssEntry)
 559     {
 560         /* Set the Limit */
 561         TssEntry->LimitLow = sizeof(KTSS) - 1;
 562         TssEntry->HighWord.Bits.LimitHi = 0;
 563     }
 564
 565     /* Now clear the I/O Map */
 566     RtlFillMemory(Tss->IoMaps[0].IoMap, 8096, -1);
 567
 568     /* Initialize Interrupt Direction Maps */
 569     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 570     RtlZeroMemory(p, 32);
 571
 572     /* Add DPMI support for interrupts */
 573     p[0] = 4;
 574     p[3] = 0x18;
 575     p[4] = 0x18;
 576
 577     /* Initialize the default Interrupt Direction Map */
 578     p = Tss->IntDirectionMap;
 579     RtlZeroMemory(Tss->IntDirectionMap, 32);
 580
 581     /* Add DPMI support */
 582     p[0] = 4;
 583     p[3] = 0x18;
 584     p[4] = 0x18;
 585 }
 586
 587 VOID
 588 NTAPI
 589 KiInitializeTSS(IN PKTSS Tss)
 590 {
 591     /* Set an invalid map base */
 592     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 593
 594     /* Disable traps during Task Switches */
 595     Tss->Flags = 0;
 596
 597     /* Set LDT and Ring 0 SS */
 598     Tss->LDT = 0;
 599     Tss->Ss0 = KGDT_R0_DATA;
 600 }
 601
 602 VOID
 603 FASTCALL
 604 Ki386InitializeTss(IN PKTSS Tss,
 605                    IN PKIDTENTRY Idt,
 606                    IN PKGDTENTRY Gdt)
 607 {
 608     PKGDTENTRY TssEntry, TaskGateEntry;
 609
 610     /* Initialize the boot TSS. */
 611     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 612     TssEntry->HighWord.Bits.Type = I386_TSS;
 613     TssEntry->HighWord.Bits.Pres = 1;
 614     TssEntry->HighWord.Bits.Dpl = 0;
 615     KiInitializeTSS2(Tss, TssEntry);
 616     KiInitializeTSS(Tss);
 617
 618     /* Load the task register */
 619     Ke386SetTr(KGDT_TSS);
 620
 621     /* Setup the Task Gate for Double Fault Traps */
 622     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 623     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 624     TaskGateEntry->HighWord.Bits.Pres = 1;
 625     TaskGateEntry->HighWord.Bits.Dpl = 0;
 626     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 627
 628     /* Initialize the TSS used for handling double faults. */
 629     Tss = (PKTSS)KiDoubleFaultTSS;
 630     KiInitializeTSS(Tss);
 631     Tss->CR3 = __readcr3();
 632     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 633     Tss->Eip = PtrToUlong(KiTrap8);
 634     Tss->Cs = KGDT_R0_CODE;
 635     Tss->Fs = KGDT_R0_PCR;
 636     Tss->Ss = Ke386GetSs();
 637     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 638     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 639
 640     /* Setup the Double Trap TSS entry in the GDT */
 641     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 642     TssEntry->HighWord.Bits.Type = I386_TSS;
 643     TssEntry->HighWord.Bits.Pres = 1;
 644     TssEntry->HighWord.Bits.Dpl = 0;
 645     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 646     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 647     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 648     TssEntry->LimitLow = KTSS_IO_MAPS;
 649
 650     /* Now setup the NMI Task Gate */
 651     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 652     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 653     TaskGateEntry->HighWord.Bits.Pres = 1;
 654     TaskGateEntry->HighWord.Bits.Dpl = 0;
 655     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 656
 657     /* Initialize the actual TSS */
 658     Tss = (PKTSS)KiNMITSS;
 659     KiInitializeTSS(Tss);
 660     Tss->CR3 = __readcr3();
 661     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 662     Tss->Eip = PtrToUlong(KiTrap2);
 663     Tss->Cs = KGDT_R0_CODE;
 664     Tss->Fs = KGDT_R0_PCR;
 665     Tss->Ss = Ke386GetSs();
 666     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 667     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 668
 669     /* And its associated TSS Entry */
 670     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 671     TssEntry->HighWord.Bits.Type = I386_TSS;
 672     TssEntry->HighWord.Bits.Pres = 1;
 673     TssEntry->HighWord.Bits.Dpl = 0;
 674     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 675     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 676     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 677     TssEntry->LimitLow = KTSS_IO_MAPS;
 678 }
 679
 680 VOID
 681 NTAPI
 682 KeFlushCurrentTb(VOID)
 683 {
 684     /* Flush the TLB by resetting CR3 */
 685     __writecr3(__readcr3());
 686 }
 687
 688 VOID
 689 NTAPI
 690 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 691 {
 692     /* Restore the CR registers */
 693     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 694     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 695     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 696     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 697
 698     //
 699     // Restore the DR registers
 700     //
 701     Ke386SetDr0(ProcessorState->SpecialRegisters.KernelDr0);
 702     Ke386SetDr1(ProcessorState->SpecialRegisters.KernelDr1);
 703     Ke386SetDr2(ProcessorState->SpecialRegisters.KernelDr2);
 704     Ke386SetDr3(ProcessorState->SpecialRegisters.KernelDr3);
 705     Ke386SetDr6(ProcessorState->SpecialRegisters.KernelDr6);
 706     Ke386SetDr7(ProcessorState->SpecialRegisters.KernelDr7);
 707
 708     //
 709     // Restore GDT, IDT, LDT and TSS
 710     //
 711     Ke386SetGlobalDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Gdtr.Limit);
 712     Ke386SetInterruptDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Idtr.Limit);
 713     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 714     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 715 }
 716
 717 VOID
 718 NTAPI
 719 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 720 {
 721     /* Save the CR registers */
 722     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 723     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 724     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 725     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 726                                            __readcr4() : 0;
 727
 728     /* Save the DR registers */
 729     ProcessorState->SpecialRegisters.KernelDr0 = Ke386GetDr0();
 730     ProcessorState->SpecialRegisters.KernelDr1 = Ke386GetDr1();
 731     ProcessorState->SpecialRegisters.KernelDr2 = Ke386GetDr2();
 732     ProcessorState->SpecialRegisters.KernelDr3 = Ke386GetDr3();
 733     ProcessorState->SpecialRegisters.KernelDr6 = Ke386GetDr6();
 734     ProcessorState->SpecialRegisters.KernelDr7 = Ke386GetDr7();
 735     Ke386SetDr7(0);
 736
 737     /* Save GDT, IDT, LDT and TSS */
 738     Ke386GetGlobalDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Gdtr.Limit);
 739     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Idtr.Limit);
 740     Ke386GetTr(ProcessorState->SpecialRegisters.Tr);
 741     Ke386GetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 742 }
 743
 744 VOID
 745 NTAPI
 746 KiInitializeMachineType(VOID)
 747 {
 748     /* Set the Machine Type we got from NTLDR */
 749     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 750 }
 751
 752 ULONG_PTR
 753 NTAPI
 754 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 755 {
 756     /* Set CS and ESP */
 757     Ke386Wrmsr(0x174, KGDT_R0_CODE, 0);
 758     Ke386Wrmsr(0x175, (ULONG)KeGetCurrentPrcb()->DpcStack, 0);
 759
 760     /* Set LSTAR */
 761     Ke386Wrmsr(0x176, (ULONG)KiFastCallEntry, 0);
 762     return 0;
 763 }
 764
 765 VOID
 766 NTAPI
 767 KiRestoreFastSyscallReturnState(VOID)
 768 {
 769     /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
 770
 771     /* Check if the CPU Supports fast system call */
 772     if (KeFeatureBits & KF_FAST_SYSCALL)
 773     {
 774         /* Do an IPI to enable it */
 775         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
 776     }
 777 }
 778
 779 ULONG_PTR
 780 NTAPI
 781 Ki386EnableDE(IN ULONG_PTR Context)
 782 {
 783     /* Enable DE */
 784     __writecr4(__readcr4() | CR4_DE);
 785     return 0;
 786 }
 787
 788 ULONG_PTR
 789 NTAPI
 790 Ki386EnableFxsr(IN ULONG_PTR Context)
 791 {
 792     /* Enable FXSR */
 793     __writecr4(__readcr4() | CR4_FXSR);
 794     return 0;
 795 }
 796
 797 ULONG_PTR
 798 NTAPI
 799 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
 800 {
 801     PKIDTENTRY IdtEntry;
 802
 803     /* Get the IDT Entry for Interrupt 19 */
 804     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[19];
 805
 806     /* Set it up */
 807     IdtEntry->Selector = KGDT_R0_CODE;
 808     IdtEntry->Offset = ((ULONG_PTR)KiTrap19 & 0xFFFF);
 809     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap19 >> 16) & 0xFFFF;
 810     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
 811     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
 812     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
 813
 814     /* Enable XMMI exceptions */
 815     __writecr4(__readcr4() | CR4_XMMEXCPT);
 816     return 0;
 817 }
 818
 819 VOID
 820 NTAPI
 821 KiI386PentiumLockErrataFixup(VOID)
 822 {
 823     KDESCRIPTOR IdtDescriptor = {0};
 824     PKIDTENTRY NewIdt, NewIdt2;
 825
 826     /* Allocate memory for a new IDT */
 827     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
 828
 829     /* Put everything after the first 7 entries on a new page */
 830     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
 831
 832     /* Disable interrupts */
 833     _disable();
 834
 835     /* Get the current IDT and copy it */
 836     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 837     RtlCopyMemory(NewIdt2,
 838                   (PVOID)IdtDescriptor.Base,
 839                   IdtDescriptor.Limit + 1);
 840     IdtDescriptor.Base = (ULONG)NewIdt2;
 841
 842     /* Set the new IDT */
 843     Ke386SetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 844     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
 845
 846     /* Restore interrupts */
 847     _enable();
 848
 849     /* Set the first 7 entries as read-only to produce a fault */
 850     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
 851 }
 852
 853 BOOLEAN
 854 NTAPI
 855 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
 856                   IN PKEXCEPTION_FRAME ExceptionFrame)
 857 {
 858     ULONG Flags = 0;
 859
 860     /* Disable interrupts and get previous state */
 861     Ke386SaveFlags(Flags);
 862     //Flags = __getcallerseflags();
 863     _disable();
 864
 865     /* Save freeze flag */
 866     KiFreezeFlag = 4;
 867
 868     /* Save the old IRQL */
 869     KiOldIrql = KeGetCurrentIrql();
 870
 871     /* Return whether interrupts were enabled */
 872     return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 873 }
 874
 875 VOID
 876 NTAPI
 877 KeThawExecution(IN BOOLEAN Enable)
 878 {
 879     /* Cleanup CPU caches */
 880     KeFlushCurrentTb();
 881
 882     /* Re-enable interrupts */
 883     if (Enable) _enable();
 884 }
 885
 886 BOOLEAN
 887 NTAPI
 888 KeInvalidateAllCaches(VOID)
 889 {
 890     /* Only supported on Pentium Pro and higher */
 891     if (KeI386CpuType < 6) return FALSE;
 892
 893     /* Invalidate all caches */
 894     __wbinvd();
 895     return TRUE;
 896 }
 897
 898 VOID
 899 FASTCALL
 900 KeZeroPages(IN PVOID Address,
 901             IN ULONG Size)
 902 {
 903     /* Not using XMMI in this routine */
 904     RtlZeroMemory(Address, Size);
 905 }
 906
 907 /* PUBLIC FUNCTIONS **********************************************************/
 908
 909 /*
 910  * @implemented
 911  */
 912 NTSTATUS
 913 NTAPI
 914 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
 915 {
 916     PFNSAVE_FORMAT FpState;
 917     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 918     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 919
 920     /* check if we are doing software emulation */
 921     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
 922
 923     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
 924     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
 925
 926     *((PVOID *) Save) = FpState;
 927 #ifdef __GNUC__
 928     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
 929 #else
 930     __asm
 931     {
 932         fnsave [FpState]
 933     };
 934 #endif
 935
 936     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
 937     return STATUS_SUCCESS;
 938 }
 939
 940 /*
 941  * @implemented
 942  */
 943 NTSTATUS
 944 NTAPI
 945 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
 946 {
 947     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
 948     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
 949     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 950
 951 #ifdef __GNUC__
 952     asm volatile("fnclex\n\t");
 953     asm volatile("frstor %0\n\t" : "=m" (*FpState));
 954 #else
 955     __asm
 956     {
 957         fnclex
 958         frstor [FpState]
 959     };
 960 #endif
 961
 962     ExFreePool(FpState);
 963     return STATUS_SUCCESS;
 964 }
 965
 966 /*
 967  * @implemented
 968  */
 969 ULONG
 970 NTAPI
 971 KeGetRecommendedSharedDataAlignment(VOID)
 972 {
 973     /* Return the global variable */
 974     return KeLargestCacheLine;
 975 }
 976
 977 /*
 978  * @implemented
 979  */
 980 VOID
 981 NTAPI
 982 KeFlushEntireTb(IN BOOLEAN Invalid,
 983                 IN BOOLEAN AllProcessors)
 984 {
 985     KIRQL OldIrql;
 986
 987     /* Raise the IRQL for the TB Flush */
 988     OldIrql = KeRaiseIrqlToSynchLevel();
 989
 990 #ifdef CONFIG_SMP
 991     /* FIXME: Support IPI Flush */
 992 #error Not yet implemented!
 993 #endif
 994
 995     /* Flush the TB for the Current CPU */
 996     KeFlushCurrentTb();
 997
 998     /* Return to Original IRQL */
 999     KeLowerIrql(OldIrql);
1000 }
1001
1002 /*
1003  * @implemented
1004  */
1005 VOID
1006 NTAPI
1007 KeSetDmaIoCoherency(IN ULONG Coherency)
1008 {
1009     /* Save the coherency globally */
1010     KiDmaIoCoherency = Coherency;
1011 }
1012
1013 /*
1014  * @implemented
1015  */
1016 KAFFINITY
1017 NTAPI
1018 KeQueryActiveProcessors(VOID)
1019 {
1020     PAGED_CODE();
1021
1022     /* Simply return the number of active processors */
1023     return KeActiveProcessors;
1024 }
1025
1026 /*
1027  * @implemented
1028  */
1029 VOID
1030 __cdecl
1031 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1032 {
1033     /* Capture the context */
1034     RtlCaptureContext(&State->ContextFrame);
1035
1036     /* Capture the control state */
1037     KiSaveProcessorControlState(State);
1038 }