reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The Boot TSS */
  18 KTSS KiBootTss;
  19
  20 /* The TSS to use for Double Fault Traps (INT 0x9) */
  21 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  22
  23 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  24 UCHAR KiNMITSS[KTSS_IO_MAPS];
  25
  26 /* The Boot GDT */
  27 KGDTENTRY KiBootGdt[256] =
  28 {
  29     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_NULL */
  30     {0xffff, 0x0000, {{0x00, 0x9b, 0xcf, 0x00}}},       /* KGDT_R0_CODE */
  31     {0xffff, 0x0000, {{0x00, 0x93, 0xcf, 0x00}}},       /* KGDT_R0_DATA */
  32     {0xffff, 0x0000, {{0x00, 0xfb, 0xcf, 0x00}}},       /* KGDT_R3_CODE */
  33     {0xffff, 0x0000, {{0x00, 0xf3, 0xcf, 0x00}}},       /* KGDT_R3_DATA*/
  34     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_TSS */
  35     {0x0001, 0xf000, {{0xdf, 0x93, 0xc0, 0xff}}},       /* KGDT_R0_PCR */
  36     {0x0fff, 0x0000, {{0x00, 0xf3, 0x40, 0x00}}},       /* KGDT_R3_TEB */
  37     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_UNUSED */
  38     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_LDT */
  39     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_DF_TSS */
  40     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}        /* KGDT_NMI_TSS */
  41 };
  42
  43 /* GDT Descriptor */
  44 KDESCRIPTOR KiGdtDescriptor = {0, sizeof(KiBootGdt) - 1, (ULONG)KiBootGdt};
  45
  46 /* CPU Features and Flags */
  47 ULONG KeI386CpuType;
  48 ULONG KeI386CpuStep;
  49 ULONG KeProcessorArchitecture;
  50 ULONG KeProcessorLevel;
  51 ULONG KeProcessorRevision;
  52 ULONG KeFeatureBits;
  53 ULONG KiFastSystemCallDisable = 1;
  54 ULONG KeI386NpxPresent = 0;
  55 ULONG KiMXCsrMask = 0;
  56 ULONG MxcsrFeatureMask = 0;
  57 ULONG KeI386XMMIPresent = 0;
  58 ULONG KeI386FxsrPresent = 0;
  59 ULONG KeI386MachineType;
  60 ULONG Ke386Pae = FALSE;
  61 ULONG Ke386NoExecute = FALSE;
  62 ULONG KeLargestCacheLine = 0x40;
  63 ULONG KeDcacheFlushCount = 0;
  64 ULONG KeIcacheFlushCount = 0;
  65 ULONG KiDmaIoCoherency = 0;
  66 CHAR KeNumberProcessors;
  67 KAFFINITY KeActiveProcessors = 1;
  68 BOOLEAN KiI386PentiumLockErrataPresent;
  69 BOOLEAN KiSMTProcessorsPresent;
  70
  71 /* Freeze data */
  72 KIRQL KiOldIrql;
  73 ULONG KiFreezeFlag;
  74
  75 /* Flush data */
  76 volatile LONG KiTbFlushTimeStamp;
  77
  78 /* CPU Signatures */
  79 static const CHAR CmpIntelID[]       = "GenuineIntel";
  80 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  81 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  82 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  83 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  84 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  85
  86 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  87
  88 VOID
  89 NTAPI
  90 CPUID(IN ULONG InfoType,
  91       OUT PULONG CpuInfoEax,
  92       OUT PULONG CpuInfoEbx,
  93       OUT PULONG CpuInfoEcx,
  94       OUT PULONG CpuInfoEdx)
  95 {
  96     ULONG CpuInfo[4];
  97
  98     /* Perform the CPUID Operation */
  99     __cpuid((int*)CpuInfo, InfoType);
 100
 101     /* Return the results */
 102     *CpuInfoEax = CpuInfo[0];
 103     *CpuInfoEbx = CpuInfo[1];
 104     *CpuInfoEcx = CpuInfo[2];
 105     *CpuInfoEdx = CpuInfo[3];
 106 }
 107
 108 VOID
 109 NTAPI
 110 WRMSR(IN ULONG Register,
 111       IN LONGLONG Value)
 112 {
 113    /* Write to the MSR */
 114     __writemsr(Register, Value);
 115 }
 116
 117 LONGLONG
 118 FASTCALL
 119 RDMSR(IN ULONG Register)
 120 {
 121     /* Read from the MSR */
 122     return __readmsr(Register);
 123 }
 124
 125 /* FUNCTIONS *****************************************************************/
 126
 127 VOID
 128 NTAPI
 129 KiSetProcessorType(VOID)
 130 {
 131     ULONG EFlags, NewEFlags;
 132     ULONG Reg, Dummy;
 133     ULONG Stepping, Type;
 134
 135     /* Start by assuming no CPUID data */
 136     KeGetCurrentPrcb()->CpuID = 0;
 137
 138     /* Save EFlags */
 139     EFlags = __readeflags();
 140
 141     /* XOR out the ID bit and update EFlags */
 142     NewEFlags = EFlags ^ EFLAGS_ID;
 143     __writeeflags(NewEFlags);
 144
 145     /* Get them back and see if they were modified */
 146     NewEFlags = __readeflags();
 147     if (NewEFlags != EFlags)
 148     {
 149         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 150         EFlags |= EFLAGS_ID;
 151         __writeeflags(EFlags);
 152
 153         /* Peform CPUID 0 to see if CPUID 1 is supported */
 154         CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
 155         if (Reg > 0)
 156         {
 157             /* Do CPUID 1 now */
 158             CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
 159
 160             /*
 161              * Get the Stepping and Type. The stepping contains both the
 162              * Model and the Step, while the Type contains the returned Type.
 163              * We ignore the family.
 164              *
 165              * For the stepping, we convert this: zzzzzzxy into this: x0y
 166              */
 167             Stepping = Reg & 0xF0;
 168             Stepping <<= 4;
 169             Stepping += (Reg & 0xFF);
 170             Stepping &= 0xF0F;
 171             Type = Reg & 0xF00;
 172             Type >>= 8;
 173
 174             /* Save them in the PRCB */
 175             KeGetCurrentPrcb()->CpuID = TRUE;
 176             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 177             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 178         }
 179         else
 180         {
 181             DPRINT1("CPUID Support lacking\n");
 182         }
 183     }
 184     else
 185     {
 186         DPRINT1("CPUID Support lacking\n");
 187     }
 188
 189     /* Restore EFLAGS */
 190     __writeeflags(EFlags);
 191 }
 192
 193 ULONG
 194 NTAPI
 195 KiGetCpuVendor(VOID)
 196 {
 197     PKPRCB Prcb = KeGetCurrentPrcb();
 198     ULONG Vendor[5];
 199     ULONG Temp;
 200
 201     /* Assume no Vendor ID and fail if no CPUID Support. */
 202     Prcb->VendorString[0] = 0;
 203     if (!Prcb->CpuID) return 0;
 204
 205     /* Get the Vendor ID and null-terminate it */
 206     CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
 207     Vendor[4] = 0;
 208
 209     /* Re-arrange vendor string */
 210     Temp = Vendor[2];
 211     Vendor[2] = Vendor[3];
 212     Vendor[3] = Temp;
 213
 214     /* Copy it to the PRCB and null-terminate it again */
 215     RtlCopyMemory(Prcb->VendorString,
 216                   &Vendor[1],
 217                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 218     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 219
 220     /* Now check the CPU Type */
 221     if (!strcmp(Prcb->VendorString, CmpIntelID))
 222     {
 223         return CPU_INTEL;
 224     }
 225     else if (!strcmp(Prcb->VendorString, CmpAmdID))
 226     {
 227         return CPU_AMD;
 228     }
 229     else if (!strcmp(Prcb->VendorString, CmpCyrixID))
 230     {
 231         DPRINT1("Cyrix CPU support not fully tested!\n");
 232         return CPU_CYRIX;
 233     }
 234     else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
 235     {
 236         DPRINT1("Transmeta CPU support not fully tested!\n");
 237         return CPU_TRANSMETA;
 238     }
 239     else if (!strcmp(Prcb->VendorString, CmpCentaurID))
 240     {
 241         DPRINT1("Centaur CPU support not fully tested!\n");
 242         return CPU_CENTAUR;
 243     }
 244     else if (!strcmp(Prcb->VendorString, CmpRiseID))
 245     {
 246         DPRINT1("Rise CPU support not fully tested!\n");
 247         return CPU_RISE;
 248     }
 249
 250     /* Invalid CPU */
 251     return 0;
 252 }
 253
 254 ULONG
 255 NTAPI
 256 KiGetFeatureBits(VOID)
 257 {
 258     PKPRCB Prcb = KeGetCurrentPrcb();
 259     ULONG Vendor;
 260     ULONG FeatureBits = KF_WORKING_PTE;
 261     ULONG Reg[4], Dummy;
 262     BOOLEAN ExtendedCPUID = TRUE;
 263     ULONG CpuFeatures = 0;
 264
 265     /* Get the Vendor ID */
 266     Vendor = KiGetCpuVendor();
 267
 268     /* Make sure we got a valid vendor ID at least. */
 269     if (!Vendor) return FeatureBits;
 270
 271     /* Get the CPUID Info. Features are in Reg[3]. */
 272     CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
 273
 274     /* Set the initial APIC ID */
 275     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 276
 277     switch (Vendor)
 278     {
 279         /* Intel CPUs */
 280         case CPU_INTEL:
 281
 282             /* Check if it's a P6 */
 283             if (Prcb->CpuType == 6)
 284             {
 285                 /* Perform the special sequence to get the MicroCode Signature */
 286                 WRMSR(0x8B, 0);
 287                 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
 288                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 289             }
 290             else if (Prcb->CpuType == 5)
 291             {
 292                 /* On P5, enable workaround for the LOCK errata. */
 293                 KiI386PentiumLockErrataPresent = TRUE;
 294             }
 295
 296             /* Check for broken P6 with bad SMP PTE implementation */
 297             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 298                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 299             {
 300                 /* Remove support for correct PTE support. */
 301                 FeatureBits &= ~KF_WORKING_PTE;
 302             }
 303
 304             /* Check if the CPU is too old to support SYSENTER */
 305             if ((Prcb->CpuType < 6) ||
 306                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 307             {
 308                 /* Disable it */
 309                 Reg[3] &= ~0x800;
 310             }
 311
 312             /* Set the current features */
 313             CpuFeatures = Reg[3];
 314
 315             break;
 316
 317         /* AMD CPUs */
 318         case CPU_AMD:
 319
 320             /* Check if this is a K5 or K6. (family 5) */
 321             if ((Reg[0] & 0x0F00) == 0x0500)
 322             {
 323                 /* Get the Model Number */
 324                 switch (Reg[0] & 0x00F0)
 325                 {
 326                     /* Model 1: K5 - 5k86 (initial models) */
 327                     case 0x0010:
 328
 329                         /* Check if this is Step 0 or 1. They don't support PGE */
 330                         if ((Reg[0] & 0x000F) > 0x03) break;
 331
 332                     /* Model 0: K5 - SSA5 */
 333                     case 0x0000:
 334
 335                         /* Model 0 doesn't support PGE at all. */
 336                         Reg[3] &= ~0x2000;
 337                         break;
 338
 339                     /* Model 8: K6-2 */
 340                     case 0x0080:
 341
 342                         /* K6-2, Step 8 and over have support for MTRR. */
 343                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 344                         break;
 345
 346                     /* Model 9: K6-III
 347                        Model D: K6-2+, K6-III+ */
 348                     case 0x0090:
 349                     case 0x00D0:
 350
 351                         FeatureBits |= KF_AMDK6MTRR;
 352                         break;
 353                 }
 354             }
 355             else if((Reg[0] & 0x0F00) < 0x0500)
 356             {
 357                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 358                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 359
 360                 /* They also don't support advanced CPUID functions. */
 361                 ExtendedCPUID = FALSE;
 362             }
 363
 364             /* Set the current features */
 365             CpuFeatures = Reg[3];
 366
 367             break;
 368
 369         /* Cyrix CPUs */
 370         case CPU_CYRIX:
 371
 372             /* FIXME: CMPXCGH8B */
 373
 374             break;
 375
 376         /* Transmeta CPUs */
 377         case CPU_TRANSMETA:
 378
 379             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 380             if ((Reg[0] & 0x0FFF) >= 0x0542)
 381             {
 382                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 383                 FeatureBits |= KF_CMPXCHG8B;
 384             }
 385
 386             break;
 387
 388         /* Centaur, IDT, Rise and VIA CPUs */
 389         case CPU_CENTAUR:
 390         case CPU_RISE:
 391
 392             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 393                However, this feature exists and operates properly without any additional steps. */
 394             FeatureBits |= KF_CMPXCHG8B;
 395
 396             break;
 397     }
 398
 399     /* Convert all CPUID Feature bits into our format */
 400     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 401     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 402     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 403     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 404     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 405     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 406     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 407     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 408     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 409     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 410     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 411     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 412     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 413     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 414
 415     /* Check if the CPU has hyper-threading */
 416     if (CpuFeatures & 0x10000000)
 417     {
 418         /* Set the number of logical CPUs */
 419         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 420         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 421         {
 422             /* We're on dual-core */
 423             KiSMTProcessorsPresent = TRUE;
 424         }
 425     }
 426     else
 427     {
 428         /* We only have a single CPU */
 429         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 430     }
 431
 432     /* Check if CPUID 0x80000000 is supported */
 433     if (ExtendedCPUID)
 434     {
 435         /* Do the call */
 436         CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
 437         if ((Reg[0] & 0xffffff00) == 0x80000000)
 438         {
 439             /* Check if CPUID 0x80000001 is supported */
 440             if (Reg[0] >= 0x80000001)
 441             {
 442                 /* Check which extended features are available. */
 443                 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
 444
 445                 /* Check if NX-bit is supported */
 446                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 447
 448                 /* Now handle each features for each CPU Vendor */
 449                 switch (Vendor)
 450                 {
 451                     case CPU_AMD:
 452                     case CPU_CENTAUR:
 453                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 454                         break;
 455                 }
 456             }
 457         }
 458     }
 459
 460     /* Return the Feature Bits */
 461     return FeatureBits;
 462 }
 463
 464 VOID
 465 NTAPI
 466 KiGetCacheInformation(VOID)
 467 {
 468     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 469     ULONG Vendor;
 470     ULONG Data[4], Dummy;
 471     ULONG CacheRequests = 0, i;
 472     ULONG CurrentRegister;
 473     UCHAR RegisterByte;
 474     BOOLEAN FirstPass = TRUE;
 475
 476     /* Set default L2 size */
 477     Pcr->SecondLevelCacheSize = 0;
 478
 479     /* Get the Vendor ID and make sure we support CPUID */
 480     Vendor = KiGetCpuVendor();
 481     if (!Vendor) return;
 482
 483     /* Check the Vendor ID */
 484     switch (Vendor)
 485     {
 486         /* Handle Intel case */
 487         case CPU_INTEL:
 488
 489             /*Check if we support CPUID 2 */
 490             CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
 491             if (Data[0] >= 2)
 492             {
 493                 /* We need to loop for the number of times CPUID will tell us to */
 494                 do
 495                 {
 496                     /* Do the CPUID call */
 497                     CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
 498
 499                     /* Check if it was the first call */
 500                     if (FirstPass)
 501                     {
 502                         /*
 503                          * The number of times to loop is the first byte. Read
 504                          * it and then destroy it so we don't get confused.
 505                          */
 506                         CacheRequests = Data[0] & 0xFF;
 507                         Data[0] &= 0xFFFFFF00;
 508
 509                         /* Don't go over this again */
 510                         FirstPass = FALSE;
 511                     }
 512
 513                     /* Loop all 4 registers */
 514                     for (i = 0; i < 4; i++)
 515                     {
 516                         /* Get the current register */
 517                         CurrentRegister = Data[i];
 518
 519                         /*
 520                          * If the upper bit is set, then this register should
 521                          * be skipped.
 522                          */
 523                         if (CurrentRegister & 0x80000000) continue;
 524
 525                         /* Keep looping for every byte inside this register */
 526                         while (CurrentRegister)
 527                         {
 528                             /* Read a byte, skip a byte. */
 529                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 530                             CurrentRegister >>= 8;
 531                             if (!RegisterByte) continue;
 532
 533                             /*
 534                              * Valid values are from 0x40 (0 bytes) to 0x49
 535                              * (32MB), or from 0x80 to 0x89 (same size but
 536                              * 8-way associative.
 537                              */
 538                             if (((RegisterByte > 0x40) &&
 539                                  (RegisterByte <= 0x49)) ||
 540                                 ((RegisterByte > 0x80) &&
 541                                 (RegisterByte <= 0x89)))
 542                             {
 543                                 /* Mask out only the first nibble */
 544                                 RegisterByte &= 0x0F;
 545
 546                                 /* Set the L2 Cache Size */
 547                                 Pcr->SecondLevelCacheSize = 0x10000 <<
 548                                                             RegisterByte;
 549                             }
 550                         }
 551                     }
 552                 } while (--CacheRequests);
 553             }
 554             break;
 555
 556         case CPU_AMD:
 557
 558             /* Check if we support CPUID 0x80000006 */
 559             CPUID(0x80000000, &Data[0], &Dummy, &Dummy, &Dummy);
 560             if (Data[0] >= 6)
 561             {
 562                 /* Get 2nd level cache and tlb size */
 563                 CPUID(0x80000006, &Dummy, &Dummy, &Data[2], &Dummy);
 564
 565                 /* Set the L2 Cache Size */
 566                 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
 567             }
 568             break;
 569
 570         case CPU_CYRIX:
 571         case CPU_TRANSMETA:
 572         case CPU_CENTAUR:
 573         case CPU_RISE:
 574
 575             /* FIXME */
 576             break;
 577     }
 578 }
 579
 580 VOID
 581 NTAPI
 582 KiSetCR0Bits(VOID)
 583 {
 584     ULONG Cr0;
 585
 586     /* Save current CR0 */
 587     Cr0 = __readcr0();
 588
 589     /* If this is a 486, enable Write-Protection */
 590     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 591
 592     /* Set new Cr0 */
 593     __writecr0(Cr0);
 594 }
 595
 596 VOID
 597 NTAPI
 598 KiInitializeTSS2(IN PKTSS Tss,
 599                  IN PKGDTENTRY TssEntry OPTIONAL)
 600 {
 601     PUCHAR p;
 602
 603     /* Make sure the GDT Entry is valid */
 604     if (TssEntry)
 605     {
 606         /* Set the Limit */
 607         TssEntry->LimitLow = sizeof(KTSS) - 1;
 608         TssEntry->HighWord.Bits.LimitHi = 0;
 609     }
 610
 611     /* Now clear the I/O Map */
 612     RtlFillMemory(Tss->IoMaps[0].IoMap, 8096, -1);
 613
 614     /* Initialize Interrupt Direction Maps */
 615     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 616     RtlZeroMemory(p, 32);
 617
 618     /* Add DPMI support for interrupts */
 619     p[0] = 4;
 620     p[3] = 0x18;
 621     p[4] = 0x18;
 622
 623     /* Initialize the default Interrupt Direction Map */
 624     p = Tss->IntDirectionMap;
 625     RtlZeroMemory(Tss->IntDirectionMap, 32);
 626
 627     /* Add DPMI support */
 628     p[0] = 4;
 629     p[3] = 0x18;
 630     p[4] = 0x18;
 631 }
 632
 633 VOID
 634 NTAPI
 635 KiInitializeTSS(IN PKTSS Tss)
 636 {
 637     /* Set an invalid map base */
 638     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 639
 640     /* Disable traps during Task Switches */
 641     Tss->Flags = 0;
 642
 643     /* Set LDT and Ring 0 SS */
 644     Tss->LDT = 0;
 645     Tss->Ss0 = KGDT_R0_DATA;
 646 }
 647
 648 VOID
 649 FASTCALL
 650 Ki386InitializeTss(IN PKTSS Tss,
 651                    IN PKIDTENTRY Idt,
 652                    IN PKGDTENTRY Gdt)
 653 {
 654     PKGDTENTRY TssEntry, TaskGateEntry;
 655
 656     /* Initialize the boot TSS. */
 657     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 658     TssEntry->HighWord.Bits.Type = I386_TSS;
 659     TssEntry->HighWord.Bits.Pres = 1;
 660     TssEntry->HighWord.Bits.Dpl = 0;
 661     KiInitializeTSS2(Tss, TssEntry);
 662     KiInitializeTSS(Tss);
 663
 664     /* Load the task register */
 665     Ke386SetTr(KGDT_TSS);
 666
 667     /* Setup the Task Gate for Double Fault Traps */
 668     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 669     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 670     TaskGateEntry->HighWord.Bits.Pres = 1;
 671     TaskGateEntry->HighWord.Bits.Dpl = 0;
 672     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 673
 674     /* Initialize the TSS used for handling double faults. */
 675     Tss = (PKTSS)KiDoubleFaultTSS;
 676     KiInitializeTSS(Tss);
 677     Tss->CR3 = __readcr3();
 678     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 679     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 680     Tss->Eip = PtrToUlong(KiTrap8);
 681     Tss->Cs = KGDT_R0_CODE;
 682     Tss->Fs = KGDT_R0_PCR;
 683     Tss->Ss = Ke386GetSs();
 684     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 685     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 686
 687     /* Setup the Double Trap TSS entry in the GDT */
 688     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 689     TssEntry->HighWord.Bits.Type = I386_TSS;
 690     TssEntry->HighWord.Bits.Pres = 1;
 691     TssEntry->HighWord.Bits.Dpl = 0;
 692     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 693     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 694     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 695     TssEntry->LimitLow = KTSS_IO_MAPS;
 696
 697     /* Now setup the NMI Task Gate */
 698     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 699     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 700     TaskGateEntry->HighWord.Bits.Pres = 1;
 701     TaskGateEntry->HighWord.Bits.Dpl = 0;
 702     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 703
 704     /* Initialize the actual TSS */
 705     Tss = (PKTSS)KiNMITSS;
 706     KiInitializeTSS(Tss);
 707     Tss->CR3 = __readcr3();
 708     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 709     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 710     Tss->Eip = PtrToUlong(KiTrap2);
 711     Tss->Cs = KGDT_R0_CODE;
 712     Tss->Fs = KGDT_R0_PCR;
 713     Tss->Ss = Ke386GetSs();
 714     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 715     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 716
 717     /* And its associated TSS Entry */
 718     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 719     TssEntry->HighWord.Bits.Type = I386_TSS;
 720     TssEntry->HighWord.Bits.Pres = 1;
 721     TssEntry->HighWord.Bits.Dpl = 0;
 722     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 723     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 724     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 725     TssEntry->LimitLow = KTSS_IO_MAPS;
 726 }
 727
 728 VOID
 729 NTAPI
 730 KeFlushCurrentTb(VOID)
 731 {
 732     /* Flush the TLB by resetting CR3 */
 733     __writecr3(__readcr3());
 734 }
 735
 736 VOID
 737 NTAPI
 738 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 739 {
 740     PKGDTENTRY TssEntry;
 741
 742     //
 743     // Restore the CR registers
 744     //
 745     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 746     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 747     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 748     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 749
 750     //
 751     // Restore the DR registers
 752     //
 753     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 754     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 755     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 756     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 757     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 758     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 759
 760     //
 761     // Restore GDT and IDT
 762     //
 763     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 764     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 765
 766     //
 767     // Clear the busy flag so we don't crash if we reload the same selector
 768     //
 769     TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
 770                             ProcessorState->SpecialRegisters.Tr);
 771     TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
 772
 773     //
 774     // Restore TSS and LDT
 775     //
 776     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 777     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 778 }
 779
 780 VOID
 781 NTAPI
 782 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 783 {
 784     /* Save the CR registers */
 785     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 786     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 787     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 788     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 789                                            __readcr4() : 0;
 790
 791     /* Save the DR registers */
 792     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 793     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 794     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 795     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 796     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 797     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 798     __writedr(7, 0);
 799
 800     /* Save GDT, IDT, LDT and TSS */
 801     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 802     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 803     ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
 804     ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
 805 }
 806
 807 VOID
 808 NTAPI
 809 KiInitializeMachineType(VOID)
 810 {
 811     /* Set the Machine Type we got from NTLDR */
 812     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 813 }
 814
 815 ULONG_PTR
 816 NTAPI
 817 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 818 {
 819     /* Set CS and ESP */
 820     WRMSR(0x174, KGDT_R0_CODE);
 821     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
 822
 823     /* Set LSTAR */
 824     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
 825     return 0;
 826 }
 827
 828 VOID
 829 NTAPI
 830 KiRestoreFastSyscallReturnState(VOID)
 831 {
 832     /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
 833
 834     /* Check if the CPU Supports fast system call */
 835     if (KeFeatureBits & KF_FAST_SYSCALL)
 836     {
 837         /* Do an IPI to enable it */
 838         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
 839     }
 840 }
 841
 842 ULONG_PTR
 843 NTAPI
 844 Ki386EnableDE(IN ULONG_PTR Context)
 845 {
 846     /* Enable DE */
 847     __writecr4(__readcr4() | CR4_DE);
 848     return 0;
 849 }
 850
 851 ULONG_PTR
 852 NTAPI
 853 Ki386EnableFxsr(IN ULONG_PTR Context)
 854 {
 855     /* Enable FXSR */
 856     __writecr4(__readcr4() | CR4_FXSR);
 857     return 0;
 858 }
 859
 860 ULONG_PTR
 861 NTAPI
 862 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
 863 {
 864     PKIDTENTRY IdtEntry;
 865
 866     /* Get the IDT Entry for Interrupt 19 */
 867     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[19];
 868
 869     /* Set it up */
 870     IdtEntry->Selector = KGDT_R0_CODE;
 871     IdtEntry->Offset = ((ULONG_PTR)KiTrap19 & 0xFFFF);
 872     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap19 >> 16) & 0xFFFF;
 873     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
 874     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
 875     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
 876
 877     /* Enable XMMI exceptions */
 878     __writecr4(__readcr4() | CR4_XMMEXCPT);
 879     return 0;
 880 }
 881
 882 VOID
 883 NTAPI
 884 KiI386PentiumLockErrataFixup(VOID)
 885 {
 886     KDESCRIPTOR IdtDescriptor;
 887     PKIDTENTRY NewIdt, NewIdt2;
 888
 889     /* Allocate memory for a new IDT */
 890     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
 891
 892     /* Put everything after the first 7 entries on a new page */
 893     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
 894
 895     /* Disable interrupts */
 896     _disable();
 897
 898     /* Get the current IDT and copy it */
 899     __sidt(&IdtDescriptor.Limit);
 900     RtlCopyMemory(NewIdt2,
 901                   (PVOID)IdtDescriptor.Base,
 902                   IdtDescriptor.Limit + 1);
 903     IdtDescriptor.Base = (ULONG)NewIdt2;
 904
 905     /* Set the new IDT */
 906     __lidt(&IdtDescriptor.Limit);
 907     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
 908
 909     /* Restore interrupts */
 910     _enable();
 911
 912     /* Set the first 7 entries as read-only to produce a fault */
 913     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
 914 }
 915
 916 BOOLEAN
 917 NTAPI
 918 KeDisableInterrupts(VOID)
 919 {
 920     ULONG Flags;
 921     BOOLEAN Return;
 922
 923     /* Get EFLAGS and check if the interrupt bit is set */
 924     Flags = __readeflags();
 925     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 926
 927     /* Disable interrupts */
 928     _disable();
 929     return Return;
 930 }
 931
 932 BOOLEAN
 933 NTAPI
 934 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
 935                   IN PKEXCEPTION_FRAME ExceptionFrame)
 936 {
 937     BOOLEAN Enable;
 938
 939     /* Disable interrupts and get previous state */
 940     Enable = KeDisableInterrupts();
 941
 942     /* Save freeze flag */
 943     KiFreezeFlag = 4;
 944
 945     /* Save the old IRQL */
 946     KiOldIrql = KeGetCurrentIrql();
 947
 948     /* Return whether interrupts were enabled */
 949     return Enable;
 950 }
 951
 952 VOID
 953 NTAPI
 954 KeThawExecution(IN BOOLEAN Enable)
 955 {
 956     /* Cleanup CPU caches */
 957     KeFlushCurrentTb();
 958
 959     /* Re-enable interrupts */
 960     if (Enable) _enable();
 961 }
 962
 963 BOOLEAN
 964 NTAPI
 965 KeInvalidateAllCaches(VOID)
 966 {
 967     /* Only supported on Pentium Pro and higher */
 968     if (KeI386CpuType < 6) return FALSE;
 969
 970     /* Invalidate all caches */
 971     __wbinvd();
 972     return TRUE;
 973 }
 974
 975 VOID
 976 FASTCALL
 977 KeZeroPages(IN PVOID Address,
 978             IN ULONG Size)
 979 {
 980     /* Not using XMMI in this routine */
 981     RtlZeroMemory(Address, Size);
 982 }
 983
 984 /* PUBLIC FUNCTIONS **********************************************************/
 985
 986 /*
 987  * @implemented
 988  */
 989 NTSTATUS
 990 NTAPI
 991 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
 992 {
 993     PFNSAVE_FORMAT FpState;
 994     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
 995     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 996
 997     /* check if we are doing software emulation */
 998     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
 999
1000     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1001     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1002
1003     *((PVOID *) Save) = FpState;
1004 #ifdef __GNUC__
1005     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1006 #else
1007     __asm
1008     {
1009         fnsave [FpState]
1010     };
1011 #endif
1012
1013     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1014     return STATUS_SUCCESS;
1015 }
1016
1017 /*
1018  * @implemented
1019  */
1020 NTSTATUS
1021 NTAPI
1022 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1023 {
1024     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1025     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1026     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1027
1028 #ifdef __GNUC__
1029     asm volatile("fnclex\n\t");
1030     asm volatile("frstor %0\n\t" : "=m" (*FpState));
1031 #else
1032     __asm
1033     {
1034         fnclex
1035         frstor [FpState]
1036     };
1037 #endif
1038
1039     ExFreePool(FpState);
1040     return STATUS_SUCCESS;
1041 }
1042
1043 /*
1044  * @implemented
1045  */
1046 ULONG
1047 NTAPI
1048 KeGetRecommendedSharedDataAlignment(VOID)
1049 {
1050     /* Return the global variable */
1051     return KeLargestCacheLine;
1052 }
1053
1054 VOID
1055 NTAPI
1056 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1057                       IN PVOID Ignored1,
1058                       IN PVOID Ignored2,
1059                       IN PVOID Ignored3)
1060 {
1061     /* Signal this packet as done */
1062     KiIpiSignalPacketDone(PacketContext);
1063
1064     /* Flush the TB for the Current CPU */
1065     KeFlushCurrentTb();
1066 }
1067
1068 /*
1069  * @implemented
1070  */
1071 VOID
1072 NTAPI
1073 KeFlushEntireTb(IN BOOLEAN Invalid,
1074                 IN BOOLEAN AllProcessors)
1075 {
1076     KIRQL OldIrql;
1077 #ifdef CONFIG_SMP
1078     KAFFINITY TargetAffinity;
1079     PKPRCB Prcb = KeGetCurrentPrcb();
1080 #endif
1081
1082     /* Raise the IRQL for the TB Flush */
1083     OldIrql = KeRaiseIrqlToSynchLevel();
1084
1085 #ifdef CONFIG_SMP
1086     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1087
1088     /* Get the current processor affinity, and exclude ourselves */
1089     TargetAffinity = KeActiveProcessors;
1090     TargetAffinity &= ~Prcb->SetMember;
1091
1092     /* Make sure this is MP */
1093     if (TargetAffinity)
1094     {
1095         /* Send an IPI TB flush to the other processors */
1096         KiIpiSendPacket(TargetAffinity,
1097                         KiFlushTargetEntireTb,
1098                         NULL,
1099                         0,
1100                         NULL);
1101     }
1102 #endif
1103
1104     /* Flush the TB for the Current CPU, and update the flush stamp */
1105     KeFlushCurrentTb();
1106
1107 #ifdef CONFIG_SMP
1108     /* If this is MP, wait for the other processors to finish */
1109     if (TargetAffinity)
1110     {
1111         /* Sanity check */
1112         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1113
1114         /* FIXME: TODO */
1115         ASSERTMSG("Not yet implemented\n", FALSE);
1116     }
1117 #endif
1118
1119     /* Update the flush stamp and return to original IRQL */
1120     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1121     KeLowerIrql(OldIrql);
1122 }
1123
1124 /*
1125  * @implemented
1126  */
1127 VOID
1128 NTAPI
1129 KeSetDmaIoCoherency(IN ULONG Coherency)
1130 {
1131     /* Save the coherency globally */
1132     KiDmaIoCoherency = Coherency;
1133 }
1134
1135 /*
1136  * @implemented
1137  */
1138 KAFFINITY
1139 NTAPI
1140 KeQueryActiveProcessors(VOID)
1141 {
1142     PAGED_CODE();
1143
1144     /* Simply return the number of active processors */
1145     return KeActiveProcessors;
1146 }
1147
1148 /*
1149  * @implemented
1150  */
1151 VOID
1152 __cdecl
1153 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1154 {
1155     /* Capture the context */
1156     RtlCaptureContext(&State->ContextFrame);
1157
1158     /* Capture the control state */
1159     KiSaveProcessorControlState(State);
1160 }