reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The Boot TSS */
  18 KTSS KiBootTss;
  19
  20 /* The TSS to use for Double Fault Traps (INT 0x9) */
  21 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  22
  23 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  24 UCHAR KiNMITSS[KTSS_IO_MAPS];
  25
  26 /* The Boot GDT */
  27 KGDTENTRY KiBootGdt[256] =
  28 {
  29     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_NULL */
  30     {0xffff, 0x0000, {{0x00, 0x9b, 0xcf, 0x00}}},       /* KGDT_R0_CODE */
  31     {0xffff, 0x0000, {{0x00, 0x93, 0xcf, 0x00}}},       /* KGDT_R0_DATA */
  32     {0xffff, 0x0000, {{0x00, 0xfb, 0xcf, 0x00}}},       /* KGDT_R3_CODE */
  33     {0xffff, 0x0000, {{0x00, 0xf3, 0xcf, 0x00}}},       /* KGDT_R3_DATA*/
  34     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_TSS */
  35     {0x0001, 0xf000, {{0xdf, 0x93, 0xc0, 0xff}}},       /* KGDT_R0_PCR */
  36     {0x0fff, 0x0000, {{0x00, 0xf3, 0x40, 0x00}}},       /* KGDT_R3_TEB */
  37     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_UNUSED */
  38     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_LDT */
  39     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_DF_TSS */
  40     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}        /* KGDT_NMI_TSS */
  41 };
  42
  43 /* GDT Descriptor */
  44 KDESCRIPTOR KiGdtDescriptor = {0, sizeof(KiBootGdt) - 1, (ULONG)KiBootGdt};
  45
  46 /* CPU Features and Flags */
  47 ULONG KeI386CpuType;
  48 ULONG KeI386CpuStep;
  49 ULONG KeProcessorArchitecture;
  50 ULONG KeProcessorLevel;
  51 ULONG KeProcessorRevision;
  52 ULONG KeFeatureBits;
  53 ULONG KiFastSystemCallDisable = 1;
  54 ULONG KeI386NpxPresent = 0;
  55 ULONG KiMXCsrMask = 0;
  56 ULONG MxcsrFeatureMask = 0;
  57 ULONG KeI386XMMIPresent = 0;
  58 ULONG KeI386FxsrPresent = 0;
  59 ULONG KeI386MachineType;
  60 ULONG Ke386Pae = FALSE;
  61 ULONG Ke386NoExecute = FALSE;
  62 ULONG KeLargestCacheLine = 0x40;
  63 ULONG KeDcacheFlushCount = 0;
  64 ULONG KeIcacheFlushCount = 0;
  65 ULONG KiDmaIoCoherency = 0;
  66 CHAR KeNumberProcessors;
  67 KAFFINITY KeActiveProcessors = 1;
  68 BOOLEAN KiI386PentiumLockErrataPresent;
  69 BOOLEAN KiSMTProcessorsPresent;
  70
  71 /* Freeze data */
  72 KIRQL KiOldIrql;
  73 ULONG KiFreezeFlag;
  74
  75 /* Flush data */
  76 volatile LONG KiTbFlushTimeStamp;
  77
  78 /* CPU Signatures */
  79 static const CHAR CmpIntelID[]       = "GenuineIntel";
  80 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  81 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  82 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  83 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  84 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  85
  86 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  87
  88 VOID
  89 NTAPI
  90 CPUID(OUT ULONG CpuInfo[4],
  91       IN ULONG InfoType)
  92 {
  93     /* Perform the CPUID Operation */
  94     __cpuid((int*)CpuInfo, InfoType);
  95 }
  96
  97 VOID
  98 NTAPI
  99 WRMSR(IN ULONG Register,
 100       IN LONGLONG Value)
 101 {
 102    /* Write to the MSR */
 103     __writemsr(Register, Value);
 104 }
 105
 106 LONGLONG
 107 NTAPI
 108 RDMSR(IN ULONG Register)
 109 {
 110     /* Read from the MSR */
 111     return __readmsr(Register);
 112 }
 113
 114 /* FUNCTIONS *****************************************************************/
 115
 116 VOID
 117 NTAPI
 118 KiSetProcessorType(VOID)
 119 {
 120     ULONG EFlags, NewEFlags;
 121     ULONG Reg[4];
 122     ULONG Stepping, Type;
 123
 124     /* Start by assuming no CPUID data */
 125     KeGetCurrentPrcb()->CpuID = 0;
 126
 127     /* Save EFlags */
 128     EFlags = __readeflags();
 129
 130     /* XOR out the ID bit and update EFlags */
 131     NewEFlags = EFlags ^ EFLAGS_ID;
 132     __writeeflags(NewEFlags);
 133
 134     /* Get them back and see if they were modified */
 135     NewEFlags = __readeflags();
 136     if (NewEFlags != EFlags)
 137     {
 138         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 139         EFlags |= EFLAGS_ID;
 140         __writeeflags(EFlags);
 141
 142         /* Peform CPUID 0 to see if CPUID 1 is supported */
 143         CPUID(Reg, 0);
 144         if (Reg[0] > 0)
 145         {
 146             /* Do CPUID 1 now */
 147             CPUID(Reg, 1);
 148
 149             /*
 150              * Get the Stepping and Type. The stepping contains both the
 151              * Model and the Step, while the Type contains the returned Type.
 152              * We ignore the family.
 153              *
 154              * For the stepping, we convert this: zzzzzzxy into this: x0y
 155              */
 156             Stepping = Reg[0] & 0xF0;
 157             Stepping <<= 4;
 158             Stepping += (Reg[0] & 0xFF);
 159             Stepping &= 0xF0F;
 160             Type = Reg[0] & 0xF00;
 161             Type >>= 8;
 162
 163             /* Save them in the PRCB */
 164             KeGetCurrentPrcb()->CpuID = TRUE;
 165             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 166             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 167         }
 168         else
 169         {
 170             DPRINT1("CPUID Support lacking\n");
 171         }
 172     }
 173     else
 174     {
 175         DPRINT1("CPUID Support lacking\n");
 176     }
 177
 178     /* Restore EFLAGS */
 179     __writeeflags(EFlags);
 180 }
 181
 182 ULONG
 183 NTAPI
 184 KiGetCpuVendor(VOID)
 185 {
 186     PKPRCB Prcb = KeGetCurrentPrcb();
 187     ULONG Vendor[5];
 188     ULONG Temp;
 189
 190     /* Assume no Vendor ID and fail if no CPUID Support. */
 191     Prcb->VendorString[0] = 0;
 192     if (!Prcb->CpuID) return 0;
 193
 194     /* Get the Vendor ID and null-terminate it */
 195     CPUID(Vendor, 0);
 196     Vendor[4] = 0;
 197
 198     /* Re-arrange vendor string */
 199     Temp = Vendor[2];
 200     Vendor[2] = Vendor[3];
 201     Vendor[3] = Temp;
 202
 203     /* Copy it to the PRCB and null-terminate it again */
 204     RtlCopyMemory(Prcb->VendorString,
 205                   &Vendor[1],
 206                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 207     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 208
 209     /* Now check the CPU Type */
 210     if (!strcmp((PCHAR)Prcb->VendorString, CmpIntelID))
 211     {
 212         return CPU_INTEL;
 213     }
 214     else if (!strcmp((PCHAR)Prcb->VendorString, CmpAmdID))
 215     {
 216         return CPU_AMD;
 217     }
 218     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCyrixID))
 219     {
 220         DPRINT1("Cyrix CPU support not fully tested!\n");
 221         return CPU_CYRIX;
 222     }
 223     else if (!strcmp((PCHAR)Prcb->VendorString, CmpTransmetaID))
 224     {
 225         DPRINT1("Transmeta CPU support not fully tested!\n");
 226         return CPU_TRANSMETA;
 227     }
 228     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCentaurID))
 229     {
 230         DPRINT1("Centaur CPU support not fully tested!\n");
 231         return CPU_CENTAUR;
 232     }
 233     else if (!strcmp((PCHAR)Prcb->VendorString, CmpRiseID))
 234     {
 235         DPRINT1("Rise CPU support not fully tested!\n");
 236         return CPU_RISE;
 237     }
 238
 239     /* Invalid CPU */
 240     return 0;
 241 }
 242
 243 ULONG
 244 NTAPI
 245 KiGetFeatureBits(VOID)
 246 {
 247     PKPRCB Prcb = KeGetCurrentPrcb();
 248     ULONG Vendor;
 249     ULONG FeatureBits = KF_WORKING_PTE;
 250     ULONG Reg[4];
 251     BOOLEAN ExtendedCPUID = TRUE;
 252     ULONG CpuFeatures = 0;
 253
 254     /* Get the Vendor ID */
 255     Vendor = KiGetCpuVendor();
 256
 257     /* Make sure we got a valid vendor ID at least. */
 258     if (!Vendor) return FeatureBits;
 259
 260     /* Get the CPUID Info. Features are in Reg[3]. */
 261     CPUID(Reg, 1);
 262
 263     /* Set the initial APIC ID */
 264     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 265
 266     switch (Vendor)
 267     {
 268         /* Intel CPUs */
 269         case CPU_INTEL:
 270             /* Check if it's a P6 */
 271             if (Prcb->CpuType == 6)
 272             {
 273                 /* Perform the special sequence to get the MicroCode Signature */
 274                 WRMSR(0x8B, 0);
 275                 CPUID(Reg, 1);
 276                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 277             }
 278             else if (Prcb->CpuType == 5)
 279             {
 280                 /* On P5, enable workaround for the LOCK errata. */
 281                 KiI386PentiumLockErrataPresent = TRUE;
 282             }
 283
 284             /* Check for broken P6 with bad SMP PTE implementation */
 285             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 286                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 287             {
 288                 /* Remove support for correct PTE support. */
 289                 FeatureBits &= ~KF_WORKING_PTE;
 290             }
 291
 292             /* Check if the CPU is too old to support SYSENTER */
 293             if ((Prcb->CpuType < 6) ||
 294                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 295             {
 296                 /* Disable it */
 297                 Reg[3] &= ~0x800;
 298             }
 299
 300             /* Set the current features */
 301             CpuFeatures = Reg[3];
 302
 303             break;
 304
 305         /* AMD CPUs */
 306         case CPU_AMD:
 307
 308             /* Check if this is a K5 or K6. (family 5) */
 309             if ((Reg[0] & 0x0F00) == 0x0500)
 310             {
 311                 /* Get the Model Number */
 312                 switch (Reg[0] & 0x00F0)
 313                 {
 314                     /* Model 1: K5 - 5k86 (initial models) */
 315                     case 0x0010:
 316
 317                         /* Check if this is Step 0 or 1. They don't support PGE */
 318                         if ((Reg[0] & 0x000F) > 0x03) break;
 319
 320                     /* Model 0: K5 - SSA5 */
 321                     case 0x0000:
 322
 323                         /* Model 0 doesn't support PGE at all. */
 324                         Reg[3] &= ~0x2000;
 325                         break;
 326
 327                     /* Model 8: K6-2 */
 328                     case 0x0080:
 329
 330                         /* K6-2, Step 8 and over have support for MTRR. */
 331                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 332                         break;
 333
 334                     /* Model 9: K6-III
 335                        Model D: K6-2+, K6-III+ */
 336                     case 0x0090:
 337                     case 0x00D0:
 338
 339                         FeatureBits |= KF_AMDK6MTRR;
 340                         break;
 341                 }
 342             }
 343             else if((Reg[0] & 0x0F00) < 0x0500)
 344             {
 345                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 346                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 347
 348                 /* They also don't support advanced CPUID functions. */
 349                 ExtendedCPUID = FALSE;
 350             }
 351
 352             /* Set the current features */
 353             CpuFeatures = Reg[3];
 354
 355             break;
 356
 357         /* Cyrix CPUs */
 358         case CPU_CYRIX:
 359             break;
 360
 361         /* Transmeta CPUs */
 362         case CPU_TRANSMETA:
 363             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 364             if ((Reg[0] & 0x0FFF) >= 0x0542)
 365             {
 366                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 367                 FeatureBits |= KF_CMPXCHG8B;
 368             }
 369
 370             break;
 371
 372         /* Centaur, IDT, Rise and VIA CPUs */
 373         case CPU_CENTAUR:
 374         case CPU_RISE:
 375             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 376                However, this feature exists and operates properly without any additional steps. */
 377             FeatureBits |= KF_CMPXCHG8B;
 378
 379             break;
 380     }
 381
 382     /* Convert all CPUID Feature bits into our format */
 383     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 384     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 385     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 386     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 387     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 388     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 389     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 390     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 391     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 392     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 393     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 394     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 395     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 396     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 397
 398     /* Check if the CPU has hyper-threading */
 399     if (CpuFeatures & 0x10000000)
 400     {
 401         /* Set the number of logical CPUs */
 402         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 403         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 404         {
 405             /* We're on dual-core */
 406             KiSMTProcessorsPresent = TRUE;
 407         }
 408     }
 409     else
 410     {
 411         /* We only have a single CPU */
 412         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 413     }
 414
 415     /* Check if CPUID 0x80000000 is supported */
 416     if (ExtendedCPUID)
 417     {
 418         /* Do the call */
 419         CPUID(Reg, 0x80000000);
 420         if ((Reg[0] & 0xffffff00) == 0x80000000)
 421         {
 422             /* Check if CPUID 0x80000001 is supported */
 423             if (Reg[0] >= 0x80000001)
 424             {
 425                 /* Check which extended features are available. */
 426                 CPUID(Reg, 0x80000001);
 427
 428                 /* Check if NX-bit is supported */
 429                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 430
 431                 /* Now handle each features for each CPU Vendor */
 432                 switch (Vendor)
 433                 {
 434                     case CPU_AMD:
 435                     case CPU_CENTAUR:
 436                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 437                         break;
 438                 }
 439             }
 440         }
 441     }
 442
 443     /* Return the Feature Bits */
 444     return FeatureBits;
 445 }
 446
 447 VOID
 448 NTAPI
 449 KiGetCacheInformation(VOID)
 450 {
 451     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 452     ULONG Vendor;
 453     ULONG Data[4];
 454     ULONG CacheRequests = 0, i;
 455     ULONG CurrentRegister;
 456     UCHAR RegisterByte;
 457     BOOLEAN FirstPass = TRUE;
 458
 459     /* Set default L2 size */
 460     Pcr->SecondLevelCacheSize = 0;
 461
 462     /* Get the Vendor ID and make sure we support CPUID */
 463     Vendor = KiGetCpuVendor();
 464     if (!Vendor) return;
 465
 466     /* Check the Vendor ID */
 467     switch (Vendor)
 468     {
 469         /* Handle Intel case */
 470         case CPU_INTEL:
 471
 472             /*Check if we support CPUID 2 */
 473             CPUID(Data, 0);
 474             if (Data[0] >= 2)
 475             {
 476                 /* We need to loop for the number of times CPUID will tell us to */
 477                 do
 478                 {
 479                     /* Do the CPUID call */
 480                     CPUID(Data, 2);
 481
 482                     /* Check if it was the first call */
 483                     if (FirstPass)
 484                     {
 485                         /*
 486                          * The number of times to loop is the first byte. Read
 487                          * it and then destroy it so we don't get confused.
 488                          */
 489                         CacheRequests = Data[0] & 0xFF;
 490                         Data[0] &= 0xFFFFFF00;
 491
 492                         /* Don't go over this again */
 493                         FirstPass = FALSE;
 494                     }
 495
 496                     /* Loop all 4 registers */
 497                     for (i = 0; i < 4; i++)
 498                     {
 499                         /* Get the current register */
 500                         CurrentRegister = Data[i];
 501
 502                         /*
 503                          * If the upper bit is set, then this register should
 504                          * be skipped.
 505                          */
 506                         if (CurrentRegister & 0x80000000) continue;
 507
 508                         /* Keep looping for every byte inside this register */
 509                         while (CurrentRegister)
 510                         {
 511                             /* Read a byte, skip a byte. */
 512                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 513                             CurrentRegister >>= 8;
 514                             if (!RegisterByte) continue;
 515
 516                             /*
 517                              * Valid values are from 0x40 (0 bytes) to 0x49
 518                              * (32MB), or from 0x80 to 0x89 (same size but
 519                              * 8-way associative.
 520                              */
 521                             if (((RegisterByte > 0x40) &&
 522                                  (RegisterByte <= 0x49)) ||
 523                                 ((RegisterByte > 0x80) &&
 524                                 (RegisterByte <= 0x89)))
 525                             {
 526                                 /* Mask out only the first nibble */
 527                                 RegisterByte &= 0x0F;
 528
 529                                 /* Set the L2 Cache Size */
 530                                 Pcr->SecondLevelCacheSize = 0x10000 <<
 531                                                             RegisterByte;
 532                             }
 533                         }
 534                     }
 535                 } while (--CacheRequests);
 536             }
 537             break;
 538
 539         case CPU_AMD:
 540
 541             /* Check if we support CPUID 0x80000006 */
 542             CPUID(Data, 0x80000000);
 543             if (Data[0] >= 6)
 544             {
 545                 /* Get 2nd level cache and tlb size */
 546                 CPUID(Data, 0x80000006);
 547
 548                 /* Set the L2 Cache Size */
 549                 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
 550             }
 551             break;
 552     }
 553 }
 554
 555 VOID
 556 NTAPI
 557 KiSetCR0Bits(VOID)
 558 {
 559     ULONG Cr0;
 560
 561     /* Save current CR0 */
 562     Cr0 = __readcr0();
 563
 564     /* If this is a 486, enable Write-Protection */
 565     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 566
 567     /* Set new Cr0 */
 568     __writecr0(Cr0);
 569 }
 570
 571 VOID
 572 NTAPI
 573 KiInitializeTSS2(IN PKTSS Tss,
 574                  IN PKGDTENTRY TssEntry OPTIONAL)
 575 {
 576     PUCHAR p;
 577
 578     /* Make sure the GDT Entry is valid */
 579     if (TssEntry)
 580     {
 581         /* Set the Limit */
 582         TssEntry->LimitLow = sizeof(KTSS) - 1;
 583         TssEntry->HighWord.Bits.LimitHi = 0;
 584     }
 585
 586     /* Now clear the I/O Map */
 587     RtlFillMemory(Tss->IoMaps[0].IoMap, 8096, -1);
 588
 589     /* Initialize Interrupt Direction Maps */
 590     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 591     RtlZeroMemory(p, 32);
 592
 593     /* Add DPMI support for interrupts */
 594     p[0] = 4;
 595     p[3] = 0x18;
 596     p[4] = 0x18;
 597
 598     /* Initialize the default Interrupt Direction Map */
 599     p = Tss->IntDirectionMap;
 600     RtlZeroMemory(Tss->IntDirectionMap, 32);
 601
 602     /* Add DPMI support */
 603     p[0] = 4;
 604     p[3] = 0x18;
 605     p[4] = 0x18;
 606 }
 607
 608 VOID
 609 NTAPI
 610 KiInitializeTSS(IN PKTSS Tss)
 611 {
 612     /* Set an invalid map base */
 613     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 614
 615     /* Disable traps during Task Switches */
 616     Tss->Flags = 0;
 617
 618     /* Set LDT and Ring 0 SS */
 619     Tss->LDT = 0;
 620     Tss->Ss0 = KGDT_R0_DATA;
 621 }
 622
 623 VOID
 624 FASTCALL
 625 Ki386InitializeTss(IN PKTSS Tss,
 626                    IN PKIDTENTRY Idt,
 627                    IN PKGDTENTRY Gdt)
 628 {
 629     PKGDTENTRY TssEntry, TaskGateEntry;
 630
 631     /* Initialize the boot TSS. */
 632     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 633     TssEntry->HighWord.Bits.Type = I386_TSS;
 634     TssEntry->HighWord.Bits.Pres = 1;
 635     TssEntry->HighWord.Bits.Dpl = 0;
 636     KiInitializeTSS2(Tss, TssEntry);
 637     KiInitializeTSS(Tss);
 638
 639     /* Load the task register */
 640     Ke386SetTr(KGDT_TSS);
 641
 642     /* Setup the Task Gate for Double Fault Traps */
 643     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 644     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 645     TaskGateEntry->HighWord.Bits.Pres = 1;
 646     TaskGateEntry->HighWord.Bits.Dpl = 0;
 647     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 648
 649     /* Initialize the TSS used for handling double faults. */
 650     Tss = (PKTSS)KiDoubleFaultTSS;
 651     KiInitializeTSS(Tss);
 652     Tss->CR3 = __readcr3();
 653     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 654     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 655     Tss->Eip = PtrToUlong(KiTrap8);
 656     Tss->Cs = KGDT_R0_CODE;
 657     Tss->Fs = KGDT_R0_PCR;
 658     Tss->Ss = Ke386GetSs();
 659     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 660     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 661
 662     /* Setup the Double Trap TSS entry in the GDT */
 663     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 664     TssEntry->HighWord.Bits.Type = I386_TSS;
 665     TssEntry->HighWord.Bits.Pres = 1;
 666     TssEntry->HighWord.Bits.Dpl = 0;
 667     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 668     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 669     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 670     TssEntry->LimitLow = KTSS_IO_MAPS;
 671
 672     /* Now setup the NMI Task Gate */
 673     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 674     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 675     TaskGateEntry->HighWord.Bits.Pres = 1;
 676     TaskGateEntry->HighWord.Bits.Dpl = 0;
 677     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 678
 679     /* Initialize the actual TSS */
 680     Tss = (PKTSS)KiNMITSS;
 681     KiInitializeTSS(Tss);
 682     Tss->CR3 = __readcr3();
 683     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 684     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 685     Tss->Eip = PtrToUlong(KiTrap2);
 686     Tss->Cs = KGDT_R0_CODE;
 687     Tss->Fs = KGDT_R0_PCR;
 688     Tss->Ss = Ke386GetSs();
 689     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 690     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 691
 692     /* And its associated TSS Entry */
 693     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 694     TssEntry->HighWord.Bits.Type = I386_TSS;
 695     TssEntry->HighWord.Bits.Pres = 1;
 696     TssEntry->HighWord.Bits.Dpl = 0;
 697     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 698     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 699     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 700     TssEntry->LimitLow = KTSS_IO_MAPS;
 701 }
 702
 703 VOID
 704 NTAPI
 705 KeFlushCurrentTb(VOID)
 706 {
 707     /* Flush the TLB by resetting CR3 */
 708     __writecr3(__readcr3());
 709 }
 710
 711 VOID
 712 NTAPI
 713 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 714 {
 715     /* Restore the CR registers */
 716     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 717     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 718     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 719     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 720
 721     //
 722     // Restore the DR registers
 723     //
 724     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 725     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 726     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 727     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 728     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 729     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 730
 731     //
 732     // Restore GDT, IDT, LDT and TSS
 733     //
 734     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 735     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 736     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 737     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 738 }
 739
 740 VOID
 741 NTAPI
 742 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 743 {
 744     /* Save the CR registers */
 745     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 746     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 747     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 748     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 749                                            __readcr4() : 0;
 750
 751     /* Save the DR registers */
 752     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 753     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 754     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 755     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 756     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 757     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 758     __writedr(7, 0);
 759
 760     /* Save GDT, IDT, LDT and TSS */
 761     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 762     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 763     Ke386GetTr(&ProcessorState->SpecialRegisters.Tr);
 764     Ke386GetLocalDescriptorTable(&ProcessorState->SpecialRegisters.Ldtr);
 765 }
 766
 767 VOID
 768 NTAPI
 769 KiInitializeMachineType(VOID)
 770 {
 771     /* Set the Machine Type we got from NTLDR */
 772     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 773 }
 774
 775 ULONG_PTR
 776 NTAPI
 777 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 778 {
 779     /* Set CS and ESP */
 780     WRMSR(0x174, KGDT_R0_CODE);
 781     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
 782
 783     /* Set LSTAR */
 784     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
 785     return 0;
 786 }
 787
 788 VOID
 789 NTAPI
 790 KiRestoreFastSyscallReturnState(VOID)
 791 {
 792     /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
 793
 794     /* Check if the CPU Supports fast system call */
 795     if (KeFeatureBits & KF_FAST_SYSCALL)
 796     {
 797         /* Do an IPI to enable it */
 798         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
 799     }
 800 }
 801
 802 ULONG_PTR
 803 NTAPI
 804 Ki386EnableDE(IN ULONG_PTR Context)
 805 {
 806     /* Enable DE */
 807     __writecr4(__readcr4() | CR4_DE);
 808     return 0;
 809 }
 810
 811 ULONG_PTR
 812 NTAPI
 813 Ki386EnableFxsr(IN ULONG_PTR Context)
 814 {
 815     /* Enable FXSR */
 816     __writecr4(__readcr4() | CR4_FXSR);
 817     return 0;
 818 }
 819
 820 ULONG_PTR
 821 NTAPI
 822 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
 823 {
 824     PKIDTENTRY IdtEntry;
 825
 826     /* Get the IDT Entry for Interrupt 19 */
 827     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[19];
 828
 829     /* Set it up */
 830     IdtEntry->Selector = KGDT_R0_CODE;
 831     IdtEntry->Offset = ((ULONG_PTR)KiTrap19 & 0xFFFF);
 832     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap19 >> 16) & 0xFFFF;
 833     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
 834     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
 835     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
 836
 837     /* Enable XMMI exceptions */
 838     __writecr4(__readcr4() | CR4_XMMEXCPT);
 839     return 0;
 840 }
 841
 842 VOID
 843 NTAPI
 844 KiI386PentiumLockErrataFixup(VOID)
 845 {
 846     KDESCRIPTOR IdtDescriptor;
 847     PKIDTENTRY NewIdt, NewIdt2;
 848
 849     /* Allocate memory for a new IDT */
 850     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
 851
 852     /* Put everything after the first 7 entries on a new page */
 853     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
 854
 855     /* Disable interrupts */
 856     _disable();
 857
 858     /* Get the current IDT and copy it */
 859     __sidt(&IdtDescriptor.Limit);
 860     RtlCopyMemory(NewIdt2,
 861                   (PVOID)IdtDescriptor.Base,
 862                   IdtDescriptor.Limit + 1);
 863     IdtDescriptor.Base = (ULONG)NewIdt2;
 864
 865     /* Set the new IDT */
 866     __lidt(&IdtDescriptor.Limit);
 867     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
 868
 869     /* Restore interrupts */
 870     _enable();
 871
 872     /* Set the first 7 entries as read-only to produce a fault */
 873     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
 874 }
 875
 876 BOOLEAN
 877 NTAPI
 878 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
 879                   IN PKEXCEPTION_FRAME ExceptionFrame)
 880 {
 881     ULONG Flags;
 882
 883     /* Disable interrupts and get previous state */
 884     Flags = __readeflags();
 885     //Flags = __getcallerseflags();
 886     _disable();
 887
 888     /* Save freeze flag */
 889     KiFreezeFlag = 4;
 890
 891     /* Save the old IRQL */
 892     KiOldIrql = KeGetCurrentIrql();
 893
 894     /* Return whether interrupts were enabled */
 895     return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 896 }
 897
 898 VOID
 899 NTAPI
 900 KeThawExecution(IN BOOLEAN Enable)
 901 {
 902     /* Cleanup CPU caches */
 903     KeFlushCurrentTb();
 904
 905     /* Re-enable interrupts */
 906     if (Enable) _enable();
 907 }
 908
 909 BOOLEAN
 910 NTAPI
 911 KeInvalidateAllCaches(VOID)
 912 {
 913     /* Only supported on Pentium Pro and higher */
 914     if (KeI386CpuType < 6) return FALSE;
 915
 916     /* Invalidate all caches */
 917     __wbinvd();
 918     return TRUE;
 919 }
 920
 921 VOID
 922 FASTCALL
 923 KeZeroPages(IN PVOID Address,
 924             IN ULONG Size)
 925 {
 926     /* Not using XMMI in this routine */
 927     RtlZeroMemory(Address, Size);
 928 }
 929
 930 /* PUBLIC FUNCTIONS **********************************************************/
 931
 932 /*
 933  * @implemented
 934  */
 935 NTSTATUS
 936 NTAPI
 937 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
 938 {
 939     PFNSAVE_FORMAT FpState;
 940     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
 941     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 942
 943     /* check if we are doing software emulation */
 944     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
 945
 946     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
 947     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
 948
 949     *((PVOID *) Save) = FpState;
 950 #ifdef __GNUC__
 951     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
 952 #else
 953     __asm
 954     {
 955         fnsave [FpState]
 956     };
 957 #endif
 958
 959     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
 960     return STATUS_SUCCESS;
 961 }
 962
 963 /*
 964  * @implemented
 965  */
 966 NTSTATUS
 967 NTAPI
 968 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
 969 {
 970     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
 971     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
 972     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 973
 974 #ifdef __GNUC__
 975     asm volatile("fnclex\n\t");
 976     asm volatile("frstor %0\n\t" : "=m" (*FpState));
 977 #else
 978     __asm
 979     {
 980         fnclex
 981         frstor [FpState]
 982     };
 983 #endif
 984
 985     ExFreePool(FpState);
 986     return STATUS_SUCCESS;
 987 }
 988
 989 /*
 990  * @implemented
 991  */
 992 ULONG
 993 NTAPI
 994 KeGetRecommendedSharedDataAlignment(VOID)
 995 {
 996     /* Return the global variable */
 997     return KeLargestCacheLine;
 998 }
 999
1000 VOID
1001 NTAPI
1002 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1003                       IN PVOID Ignored1,
1004                       IN PVOID Ignored2,
1005                       IN PVOID Ignored3)
1006 {
1007     /* Signal this packet as done */
1008     KiIpiSignalPacketDone(PacketContext);
1009
1010     /* Flush the TB for the Current CPU */
1011     KeFlushCurrentTb();
1012 }
1013
1014 /*
1015  * @implemented
1016  */
1017 VOID
1018 NTAPI
1019 KeFlushEntireTb(IN BOOLEAN Invalid,
1020                 IN BOOLEAN AllProcessors)
1021 {
1022     KIRQL OldIrql;
1023 #ifdef CONFIG_SMP
1024     KAFFINITY TargetAffinity;
1025     PKPRCB Prcb = KeGetCurrentPrcb();
1026 #endif
1027
1028     /* Raise the IRQL for the TB Flush */
1029     OldIrql = KeRaiseIrqlToSynchLevel();
1030
1031 #ifdef CONFIG_SMP
1032     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1033
1034     /* Get the current processor affinity, and exclude ourselves */
1035     TargetAffinity = KeActiveProcessors;
1036     TargetAffinity &= ~Prcb->SetMember;
1037
1038     /* Make sure this is MP */
1039     if (TargetAffinity)
1040     {
1041         /* Send an IPI TB flush to the other processors */
1042         KiIpiSendPacket(TargetAffinity,
1043                         KiFlushTargetEntireTb,
1044                         NULL,
1045                         0,
1046                         NULL);
1047     }
1048 #endif
1049
1050     /* Flush the TB for the Current CPU, and update the flush stamp */
1051     KeFlushCurrentTb();
1052
1053 #ifdef CONFIG_SMP
1054     /* If this is MP, wait for the other processors to finish */
1055     if (TargetAffinity)
1056     {
1057         /* Sanity check */
1058         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1059
1060         /* FIXME: TODO */
1061         ASSERTMSG("Not yet implemented\n", FALSE);
1062     }
1063 #endif
1064
1065     /* Update the flush stamp and return to original IRQL */
1066     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1067     KeLowerIrql(OldIrql);
1068 }
1069
1070 /*
1071  * @implemented
1072  */
1073 VOID
1074 NTAPI
1075 KeSetDmaIoCoherency(IN ULONG Coherency)
1076 {
1077     /* Save the coherency globally */
1078     KiDmaIoCoherency = Coherency;
1079 }
1080
1081 /*
1082  * @implemented
1083  */
1084 KAFFINITY
1085 NTAPI
1086 KeQueryActiveProcessors(VOID)
1087 {
1088     PAGED_CODE();
1089
1090     /* Simply return the number of active processors */
1091     return KeActiveProcessors;
1092 }
1093
1094 /*
1095  * @implemented
1096  */
1097 VOID
1098 __cdecl
1099 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1100 {
1101     /* Capture the context */
1102     RtlCaptureContext(&State->ContextFrame);
1103
1104     /* Capture the control state */
1105     KiSaveProcessorControlState(State);
1106 }