reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The TSS to use for Double Fault Traps (INT 0x9) */
  18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  19
  20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  21 UCHAR KiNMITSS[KTSS_IO_MAPS];
  22
  23 /* CPU Features and Flags */
  24 ULONG KeI386CpuType;
  25 ULONG KeI386CpuStep;
  26 ULONG KeProcessorArchitecture;
  27 ULONG KeProcessorLevel;
  28 ULONG KeProcessorRevision;
  29 ULONG KeFeatureBits;
  30 ULONG KiFastSystemCallDisable = 1;
  31 ULONG KeI386NpxPresent = 0;
  32 ULONG KiMXCsrMask = 0;
  33 ULONG MxcsrFeatureMask = 0;
  34 ULONG KeI386XMMIPresent = 0;
  35 ULONG KeI386FxsrPresent = 0;
  36 ULONG KeI386MachineType;
  37 ULONG Ke386Pae = FALSE;
  38 ULONG Ke386NoExecute = FALSE;
  39 ULONG KeLargestCacheLine = 0x40;
  40 ULONG KeDcacheFlushCount = 0;
  41 ULONG KeIcacheFlushCount = 0;
  42 ULONG KiDmaIoCoherency = 0;
  43 CHAR KeNumberProcessors;
  44 KAFFINITY KeActiveProcessors = 1;
  45 BOOLEAN KiI386PentiumLockErrataPresent;
  46 BOOLEAN KiSMTProcessorsPresent;
  47
  48 /* Flush data */
  49 volatile LONG KiTbFlushTimeStamp;
  50
  51 /* CPU Signatures */
  52 static const CHAR CmpIntelID[]       = "GenuineIntel";
  53 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  54 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  55 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  56 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  57 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  58
  59 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  60
  61 VOID
  62 NTAPI
  63 CPUID(IN ULONG InfoType,
  64       OUT PULONG CpuInfoEax,
  65       OUT PULONG CpuInfoEbx,
  66       OUT PULONG CpuInfoEcx,
  67       OUT PULONG CpuInfoEdx)
  68 {
  69     ULONG CpuInfo[4];
  70
  71     /* Perform the CPUID Operation */
  72     __cpuid((int*)CpuInfo, InfoType);
  73
  74     /* Return the results */
  75     *CpuInfoEax = CpuInfo[0];
  76     *CpuInfoEbx = CpuInfo[1];
  77     *CpuInfoEcx = CpuInfo[2];
  78     *CpuInfoEdx = CpuInfo[3];
  79 }
  80
  81 VOID
  82 NTAPI
  83 WRMSR(IN ULONG Register,
  84       IN LONGLONG Value)
  85 {
  86    /* Write to the MSR */
  87     __writemsr(Register, Value);
  88 }
  89
  90 LONGLONG
  91 FASTCALL
  92 RDMSR(IN ULONG Register)
  93 {
  94     /* Read from the MSR */
  95     return __readmsr(Register);
  96 }
  97
  98 /* FUNCTIONS *****************************************************************/
  99
 100 VOID
 101 NTAPI
 102 KiSetProcessorType(VOID)
 103 {
 104     ULONG EFlags, NewEFlags;
 105     ULONG Reg, Dummy;
 106     ULONG Stepping, Type;
 107
 108     /* Start by assuming no CPUID data */
 109     KeGetCurrentPrcb()->CpuID = 0;
 110
 111     /* Save EFlags */
 112     EFlags = __readeflags();
 113
 114     /* XOR out the ID bit and update EFlags */
 115     NewEFlags = EFlags ^ EFLAGS_ID;
 116     __writeeflags(NewEFlags);
 117
 118     /* Get them back and see if they were modified */
 119     NewEFlags = __readeflags();
 120     if (NewEFlags != EFlags)
 121     {
 122         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 123         EFlags |= EFLAGS_ID;
 124         __writeeflags(EFlags);
 125
 126         /* Peform CPUID 0 to see if CPUID 1 is supported */
 127         CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
 128         if (Reg > 0)
 129         {
 130             /* Do CPUID 1 now */
 131             CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
 132
 133             /*
 134              * Get the Stepping and Type. The stepping contains both the
 135              * Model and the Step, while the Type contains the returned Type.
 136              * We ignore the family.
 137              *
 138              * For the stepping, we convert this: zzzzzzxy into this: x0y
 139              */
 140             Stepping = Reg & 0xF0;
 141             Stepping <<= 4;
 142             Stepping += (Reg & 0xFF);
 143             Stepping &= 0xF0F;
 144             Type = Reg & 0xF00;
 145             Type >>= 8;
 146
 147             /* Save them in the PRCB */
 148             KeGetCurrentPrcb()->CpuID = TRUE;
 149             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 150             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 151         }
 152         else
 153         {
 154             DPRINT1("CPUID Support lacking\n");
 155         }
 156     }
 157     else
 158     {
 159         DPRINT1("CPUID Support lacking\n");
 160     }
 161
 162     /* Restore EFLAGS */
 163     __writeeflags(EFlags);
 164 }
 165
 166 ULONG
 167 NTAPI
 168 KiGetCpuVendor(VOID)
 169 {
 170     PKPRCB Prcb = KeGetCurrentPrcb();
 171     ULONG Vendor[5];
 172     ULONG Temp;
 173
 174     /* Assume no Vendor ID and fail if no CPUID Support. */
 175     Prcb->VendorString[0] = 0;
 176     if (!Prcb->CpuID) return 0;
 177
 178     /* Get the Vendor ID and null-terminate it */
 179     CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
 180     Vendor[4] = 0;
 181
 182     /* Re-arrange vendor string */
 183     Temp = Vendor[2];
 184     Vendor[2] = Vendor[3];
 185     Vendor[3] = Temp;
 186
 187     /* Copy it to the PRCB and null-terminate it again */
 188     RtlCopyMemory(Prcb->VendorString,
 189                   &Vendor[1],
 190                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 191     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 192
 193     /* Now check the CPU Type */
 194     if (!strcmp((PCHAR)Prcb->VendorString, CmpIntelID))
 195     {
 196         return CPU_INTEL;
 197     }
 198     else if (!strcmp((PCHAR)Prcb->VendorString, CmpAmdID))
 199     {
 200         return CPU_AMD;
 201     }
 202     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCyrixID))
 203     {
 204         DPRINT1("Cyrix CPU support not fully tested!\n");
 205         return CPU_CYRIX;
 206     }
 207     else if (!strcmp((PCHAR)Prcb->VendorString, CmpTransmetaID))
 208     {
 209         DPRINT1("Transmeta CPU support not fully tested!\n");
 210         return CPU_TRANSMETA;
 211     }
 212     else if (!strcmp((PCHAR)Prcb->VendorString, CmpCentaurID))
 213     {
 214         DPRINT1("Centaur CPU support not fully tested!\n");
 215         return CPU_CENTAUR;
 216     }
 217     else if (!strcmp((PCHAR)Prcb->VendorString, CmpRiseID))
 218     {
 219         DPRINT1("Rise CPU support not fully tested!\n");
 220         return CPU_RISE;
 221     }
 222
 223     /* Invalid CPU */
 224     return 0;
 225 }
 226
 227 ULONG
 228 NTAPI
 229 KiGetFeatureBits(VOID)
 230 {
 231     PKPRCB Prcb = KeGetCurrentPrcb();
 232     ULONG Vendor;
 233     ULONG FeatureBits = KF_WORKING_PTE;
 234     ULONG Reg[4], Dummy;
 235     BOOLEAN ExtendedCPUID = TRUE;
 236     ULONG CpuFeatures = 0;
 237
 238     /* Get the Vendor ID */
 239     Vendor = KiGetCpuVendor();
 240
 241     /* Make sure we got a valid vendor ID at least. */
 242     if (!Vendor) return FeatureBits;
 243
 244     /* Get the CPUID Info. Features are in Reg[3]. */
 245     CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
 246
 247     /* Set the initial APIC ID */
 248     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 249
 250     switch (Vendor)
 251     {
 252         /* Intel CPUs */
 253         case CPU_INTEL:
 254
 255             /* Check if it's a P6 */
 256             if (Prcb->CpuType == 6)
 257             {
 258                 /* Perform the special sequence to get the MicroCode Signature */
 259                 WRMSR(0x8B, 0);
 260                 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
 261                 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 262             }
 263             else if (Prcb->CpuType == 5)
 264             {
 265                 /* On P5, enable workaround for the LOCK errata. */
 266                 KiI386PentiumLockErrataPresent = TRUE;
 267             }
 268
 269             /* Check for broken P6 with bad SMP PTE implementation */
 270             if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 271                 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 272             {
 273                 /* Remove support for correct PTE support. */
 274                 FeatureBits &= ~KF_WORKING_PTE;
 275             }
 276
 277             /* Check if the CPU is too old to support SYSENTER */
 278             if ((Prcb->CpuType < 6) ||
 279                 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 280             {
 281                 /* Disable it */
 282                 Reg[3] &= ~0x800;
 283             }
 284
 285             /* Set the current features */
 286             CpuFeatures = Reg[3];
 287
 288             break;
 289
 290         /* AMD CPUs */
 291         case CPU_AMD:
 292
 293             /* Check if this is a K5 or K6. (family 5) */
 294             if ((Reg[0] & 0x0F00) == 0x0500)
 295             {
 296                 /* Get the Model Number */
 297                 switch (Reg[0] & 0x00F0)
 298                 {
 299                     /* Model 1: K5 - 5k86 (initial models) */
 300                     case 0x0010:
 301
 302                         /* Check if this is Step 0 or 1. They don't support PGE */
 303                         if ((Reg[0] & 0x000F) > 0x03) break;
 304
 305                     /* Model 0: K5 - SSA5 */
 306                     case 0x0000:
 307
 308                         /* Model 0 doesn't support PGE at all. */
 309                         Reg[3] &= ~0x2000;
 310                         break;
 311
 312                     /* Model 8: K6-2 */
 313                     case 0x0080:
 314
 315                         /* K6-2, Step 8 and over have support for MTRR. */
 316                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 317                         break;
 318
 319                     /* Model 9: K6-III
 320                        Model D: K6-2+, K6-III+ */
 321                     case 0x0090:
 322                     case 0x00D0:
 323
 324                         FeatureBits |= KF_AMDK6MTRR;
 325                         break;
 326                 }
 327             }
 328             else if((Reg[0] & 0x0F00) < 0x0500)
 329             {
 330                 /* Families below 5 don't support PGE, PSE or CMOV at all */
 331                 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 332
 333                 /* They also don't support advanced CPUID functions. */
 334                 ExtendedCPUID = FALSE;
 335             }
 336
 337             /* Set the current features */
 338             CpuFeatures = Reg[3];
 339
 340             break;
 341
 342         /* Cyrix CPUs */
 343         case CPU_CYRIX:
 344
 345             /* FIXME: CMPXCGH8B */
 346
 347             break;
 348
 349         /* Transmeta CPUs */
 350         case CPU_TRANSMETA:
 351
 352             /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 353             if ((Reg[0] & 0x0FFF) >= 0x0542)
 354             {
 355                 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 356                 FeatureBits |= KF_CMPXCHG8B;
 357             }
 358
 359             break;
 360
 361         /* Centaur, IDT, Rise and VIA CPUs */
 362         case CPU_CENTAUR:
 363         case CPU_RISE:
 364
 365             /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
 366                However, this feature exists and operates properly without any additional steps. */
 367             FeatureBits |= KF_CMPXCHG8B;
 368
 369             break;
 370     }
 371
 372     /* Convert all CPUID Feature bits into our format */
 373     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 374     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 375     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 376     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 377     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 378     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 379     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 380     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 381     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 382     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 383     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 384     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 385     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 386     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 387
 388     /* Check if the CPU has hyper-threading */
 389     if (CpuFeatures & 0x10000000)
 390     {
 391         /* Set the number of logical CPUs */
 392         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 393         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 394         {
 395             /* We're on dual-core */
 396             KiSMTProcessorsPresent = TRUE;
 397         }
 398     }
 399     else
 400     {
 401         /* We only have a single CPU */
 402         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 403     }
 404
 405     /* Check if CPUID 0x80000000 is supported */
 406     if (ExtendedCPUID)
 407     {
 408         /* Do the call */
 409         CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
 410         if ((Reg[0] & 0xffffff00) == 0x80000000)
 411         {
 412             /* Check if CPUID 0x80000001 is supported */
 413             if (Reg[0] >= 0x80000001)
 414             {
 415                 /* Check which extended features are available. */
 416                 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
 417
 418                 /* Check if NX-bit is supported */
 419                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 420
 421                 /* Now handle each features for each CPU Vendor */
 422                 switch (Vendor)
 423                 {
 424                     case CPU_AMD:
 425                     case CPU_CENTAUR:
 426                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 427                         break;
 428                 }
 429             }
 430         }
 431     }
 432
 433     /* Return the Feature Bits */
 434     return FeatureBits;
 435 }
 436
 437 VOID
 438 NTAPI
 439 KiGetCacheInformation(VOID)
 440 {
 441     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 442     ULONG Vendor;
 443     ULONG Data[4], Dummy;
 444     ULONG CacheRequests = 0, i;
 445     ULONG CurrentRegister;
 446     UCHAR RegisterByte;
 447     BOOLEAN FirstPass = TRUE;
 448
 449     /* Set default L2 size */
 450     Pcr->SecondLevelCacheSize = 0;
 451
 452     /* Get the Vendor ID and make sure we support CPUID */
 453     Vendor = KiGetCpuVendor();
 454     if (!Vendor) return;
 455
 456     /* Check the Vendor ID */
 457     switch (Vendor)
 458     {
 459         /* Handle Intel case */
 460         case CPU_INTEL:
 461
 462             /*Check if we support CPUID 2 */
 463             CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
 464             if (Data[0] >= 2)
 465             {
 466                 /* We need to loop for the number of times CPUID will tell us to */
 467                 do
 468                 {
 469                     /* Do the CPUID call */
 470                     CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
 471
 472                     /* Check if it was the first call */
 473                     if (FirstPass)
 474                     {
 475                         /*
 476                          * The number of times to loop is the first byte. Read
 477                          * it and then destroy it so we don't get confused.
 478                          */
 479                         CacheRequests = Data[0] & 0xFF;
 480                         Data[0] &= 0xFFFFFF00;
 481
 482                         /* Don't go over this again */
 483                         FirstPass = FALSE;
 484                     }
 485
 486                     /* Loop all 4 registers */
 487                     for (i = 0; i < 4; i++)
 488                     {
 489                         /* Get the current register */
 490                         CurrentRegister = Data[i];
 491
 492                         /*
 493                          * If the upper bit is set, then this register should
 494                          * be skipped.
 495                          */
 496                         if (CurrentRegister & 0x80000000) continue;
 497
 498                         /* Keep looping for every byte inside this register */
 499                         while (CurrentRegister)
 500                         {
 501                             /* Read a byte, skip a byte. */
 502                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 503                             CurrentRegister >>= 8;
 504                             if (!RegisterByte) continue;
 505
 506                             /*
 507                              * Valid values are from 0x40 (0 bytes) to 0x49
 508                              * (32MB), or from 0x80 to 0x89 (same size but
 509                              * 8-way associative.
 510                              */
 511                             if (((RegisterByte > 0x40) &&
 512                                  (RegisterByte <= 0x49)) ||
 513                                 ((RegisterByte > 0x80) &&
 514                                 (RegisterByte <= 0x89)))
 515                             {
 516                                 /* Mask out only the first nibble */
 517                                 RegisterByte &= 0x0F;
 518
 519                                 /* Set the L2 Cache Size */
 520                                 Pcr->SecondLevelCacheSize = 0x10000 <<
 521                                                             RegisterByte;
 522                             }
 523                         }
 524                     }
 525                 } while (--CacheRequests);
 526             }
 527             break;
 528
 529         case CPU_AMD:
 530
 531             /* Check if we support CPUID 0x80000006 */
 532             CPUID(0x80000000, &Data[0], &Dummy, &Dummy, &Dummy);
 533             if (Data[0] >= 6)
 534             {
 535                 /* Get 2nd level cache and tlb size */
 536                 CPUID(0x80000006, &Dummy, &Dummy, &Data[2], &Dummy);
 537
 538                 /* Set the L2 Cache Size */
 539                 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
 540             }
 541             break;
 542
 543         case CPU_CYRIX:
 544         case CPU_TRANSMETA:
 545         case CPU_CENTAUR:
 546         case CPU_RISE:
 547
 548             /* FIXME */
 549             break;
 550     }
 551 }
 552
 553 VOID
 554 NTAPI
 555 KiSetCR0Bits(VOID)
 556 {
 557     ULONG Cr0;
 558
 559     /* Save current CR0 */
 560     Cr0 = __readcr0();
 561
 562     /* If this is a 486, enable Write-Protection */
 563     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 564
 565     /* Set new Cr0 */
 566     __writecr0(Cr0);
 567 }
 568
 569 VOID
 570 NTAPI
 571 KiInitializeTSS2(IN PKTSS Tss,
 572                  IN PKGDTENTRY TssEntry OPTIONAL)
 573 {
 574     PUCHAR p;
 575
 576     /* Make sure the GDT Entry is valid */
 577     if (TssEntry)
 578     {
 579         /* Set the Limit */
 580         TssEntry->LimitLow = sizeof(KTSS) - 1;
 581         TssEntry->HighWord.Bits.LimitHi = 0;
 582     }
 583
 584     /* Now clear the I/O Map */
 585     ASSERT(IOPM_COUNT == 1);
 586     RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
 587
 588     /* Initialize Interrupt Direction Maps */
 589     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 590     RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
 591
 592     /* Add DPMI support for interrupts */
 593     p[0] = 4;
 594     p[3] = 0x18;
 595     p[4] = 0x18;
 596
 597     /* Initialize the default Interrupt Direction Map */
 598     p = Tss->IntDirectionMap;
 599     RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
 600
 601     /* Add DPMI support */
 602     p[0] = 4;
 603     p[3] = 0x18;
 604     p[4] = 0x18;
 605 }
 606
 607 VOID
 608 NTAPI
 609 KiInitializeTSS(IN PKTSS Tss)
 610 {
 611     /* Set an invalid map base */
 612     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 613
 614     /* Disable traps during Task Switches */
 615     Tss->Flags = 0;
 616
 617     /* Set LDT and Ring 0 SS */
 618     Tss->LDT = 0;
 619     Tss->Ss0 = KGDT_R0_DATA;
 620 }
 621
 622 VOID
 623 FASTCALL
 624 Ki386InitializeTss(IN PKTSS Tss,
 625                    IN PKIDTENTRY Idt,
 626                    IN PKGDTENTRY Gdt)
 627 {
 628     PKGDTENTRY TssEntry, TaskGateEntry;
 629
 630     /* Initialize the boot TSS. */
 631     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 632     TssEntry->HighWord.Bits.Type = I386_TSS;
 633     TssEntry->HighWord.Bits.Pres = 1;
 634     TssEntry->HighWord.Bits.Dpl = 0;
 635     KiInitializeTSS2(Tss, TssEntry);
 636     KiInitializeTSS(Tss);
 637
 638     /* Load the task register */
 639     Ke386SetTr(KGDT_TSS);
 640
 641     /* Setup the Task Gate for Double Fault Traps */
 642     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 643     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 644     TaskGateEntry->HighWord.Bits.Pres = 1;
 645     TaskGateEntry->HighWord.Bits.Dpl = 0;
 646     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 647
 648     /* Initialize the TSS used for handling double faults. */
 649     Tss = (PKTSS)KiDoubleFaultTSS;
 650     KiInitializeTSS(Tss);
 651     Tss->CR3 = __readcr3();
 652     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 653     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 654     Tss->Eip = PtrToUlong(KiTrap08);
 655     Tss->Cs = KGDT_R0_CODE;
 656     Tss->Fs = KGDT_R0_PCR;
 657     Tss->Ss = Ke386GetSs();
 658     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 659     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 660
 661     /* Setup the Double Trap TSS entry in the GDT */
 662     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 663     TssEntry->HighWord.Bits.Type = I386_TSS;
 664     TssEntry->HighWord.Bits.Pres = 1;
 665     TssEntry->HighWord.Bits.Dpl = 0;
 666     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 667     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 668     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 669     TssEntry->LimitLow = KTSS_IO_MAPS;
 670
 671     /* Now setup the NMI Task Gate */
 672     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 673     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 674     TaskGateEntry->HighWord.Bits.Pres = 1;
 675     TaskGateEntry->HighWord.Bits.Dpl = 0;
 676     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 677
 678     /* Initialize the actual TSS */
 679     Tss = (PKTSS)KiNMITSS;
 680     KiInitializeTSS(Tss);
 681     Tss->CR3 = __readcr3();
 682     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 683     Tss->Esp = PtrToUlong(KiDoubleFaultStack);
 684     Tss->Eip = PtrToUlong(KiTrap02);
 685     Tss->Cs = KGDT_R0_CODE;
 686     Tss->Fs = KGDT_R0_PCR;
 687     Tss->Ss = Ke386GetSs();
 688     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 689     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 690
 691     /* And its associated TSS Entry */
 692     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 693     TssEntry->HighWord.Bits.Type = I386_TSS;
 694     TssEntry->HighWord.Bits.Pres = 1;
 695     TssEntry->HighWord.Bits.Dpl = 0;
 696     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 697     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 698     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 699     TssEntry->LimitLow = KTSS_IO_MAPS;
 700 }
 701
 702 VOID
 703 NTAPI
 704 KeFlushCurrentTb(VOID)
 705 {
 706     /* Flush the TLB by resetting CR3 */
 707     __writecr3(__readcr3());
 708 }
 709
 710 VOID
 711 NTAPI
 712 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 713 {
 714     PKGDTENTRY TssEntry;
 715
 716     //
 717     // Restore the CR registers
 718     //
 719     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 720     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 721     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 722     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 723
 724     //
 725     // Restore the DR registers
 726     //
 727     __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
 728     __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
 729     __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
 730     __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
 731     __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
 732     __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
 733
 734     //
 735     // Restore GDT and IDT
 736     //
 737     Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 738     __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 739
 740     //
 741     // Clear the busy flag so we don't crash if we reload the same selector
 742     //
 743     TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
 744                             ProcessorState->SpecialRegisters.Tr);
 745     TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
 746
 747     //
 748     // Restore TSS and LDT
 749     //
 750     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 751     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 752 }
 753
 754 VOID
 755 NTAPI
 756 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 757 {
 758     /* Save the CR registers */
 759     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 760     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 761     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 762     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 763                                            __readcr4() : 0;
 764
 765     /* Save the DR registers */
 766     ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
 767     ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
 768     ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
 769     ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
 770     ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
 771     ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
 772     __writedr(7, 0);
 773
 774     /* Save GDT, IDT, LDT and TSS */
 775     Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
 776     __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
 777     ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
 778     ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
 779 }
 780
 781 VOID
 782 NTAPI
 783 KiInitializeMachineType(VOID)
 784 {
 785     /* Set the Machine Type we got from NTLDR */
 786     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 787 }
 788
 789 ULONG_PTR
 790 NTAPI
 791 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 792 {
 793     /* Set CS and ESP */
 794     WRMSR(0x174, KGDT_R0_CODE);
 795     WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
 796
 797     /* Set LSTAR */
 798     WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
 799     return 0;
 800 }
 801
 802 VOID
 803 NTAPI
 804 KiRestoreFastSyscallReturnState(VOID)
 805 {
 806     /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
 807
 808     /* Check if the CPU Supports fast system call */
 809     if (KeFeatureBits & KF_FAST_SYSCALL)
 810     {
 811         /* Do an IPI to enable it */
 812         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
 813     }
 814 }
 815
 816 ULONG_PTR
 817 NTAPI
 818 Ki386EnableDE(IN ULONG_PTR Context)
 819 {
 820     /* Enable DE */
 821     __writecr4(__readcr4() | CR4_DE);
 822     return 0;
 823 }
 824
 825 ULONG_PTR
 826 NTAPI
 827 Ki386EnableFxsr(IN ULONG_PTR Context)
 828 {
 829     /* Enable FXSR */
 830     __writecr4(__readcr4() | CR4_FXSR);
 831     return 0;
 832 }
 833
 834 ULONG_PTR
 835 NTAPI
 836 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
 837 {
 838     PKIDTENTRY IdtEntry;
 839
 840     /* Get the IDT Entry for Interrupt 0x13 */
 841     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
 842
 843     /* Set it up */
 844     IdtEntry->Selector = KGDT_R0_CODE;
 845     IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
 846     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
 847     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
 848     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
 849     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
 850
 851     /* Enable XMMI exceptions */
 852     __writecr4(__readcr4() | CR4_XMMEXCPT);
 853     return 0;
 854 }
 855
 856 VOID
 857 NTAPI
 858 KiI386PentiumLockErrataFixup(VOID)
 859 {
 860     KDESCRIPTOR IdtDescriptor;
 861     PKIDTENTRY NewIdt, NewIdt2;
 862
 863     /* Allocate memory for a new IDT */
 864     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
 865
 866     /* Put everything after the first 7 entries on a new page */
 867     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
 868
 869     /* Disable interrupts */
 870     _disable();
 871
 872     /* Get the current IDT and copy it */
 873     __sidt(&IdtDescriptor.Limit);
 874     RtlCopyMemory(NewIdt2,
 875                   (PVOID)IdtDescriptor.Base,
 876                   IdtDescriptor.Limit + 1);
 877     IdtDescriptor.Base = (ULONG)NewIdt2;
 878
 879     /* Set the new IDT */
 880     __lidt(&IdtDescriptor.Limit);
 881     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
 882
 883     /* Restore interrupts */
 884     _enable();
 885
 886     /* Set the first 7 entries as read-only to produce a fault */
 887     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
 888 }
 889
 890 BOOLEAN
 891 NTAPI
 892 KeDisableInterrupts(VOID)
 893 {
 894     ULONG Flags;
 895     BOOLEAN Return;
 896
 897     /* Get EFLAGS and check if the interrupt bit is set */
 898     Flags = __readeflags();
 899     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 900
 901     /* Disable interrupts */
 902     _disable();
 903     return Return;
 904 }
 905
 906 BOOLEAN
 907 NTAPI
 908 KeInvalidateAllCaches(VOID)
 909 {
 910     /* Only supported on Pentium Pro and higher */
 911     if (KeI386CpuType < 6) return FALSE;
 912
 913     /* Invalidate all caches */
 914     __wbinvd();
 915     return TRUE;
 916 }
 917
 918 VOID
 919 FASTCALL
 920 KeZeroPages(IN PVOID Address,
 921             IN ULONG Size)
 922 {
 923     /* Not using XMMI in this routine */
 924     RtlZeroMemory(Address, Size);
 925 }
 926
 927 VOID
 928 NTAPI
 929 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
 930                      IN PKEXCEPTION_FRAME ExceptionFrame)
 931 {
 932     PKPRCB Prcb = KeGetCurrentPrcb();
 933
 934     //
 935     // Save full context
 936     //
 937     Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
 938                                                      CONTEXT_DEBUG_REGISTERS;
 939     KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
 940
 941     //
 942     // Save control registers
 943     //
 944     KiSaveProcessorControlState(&Prcb->ProcessorState);
 945 }
 946
 947 /* PUBLIC FUNCTIONS **********************************************************/
 948
 949 /*
 950  * @implemented
 951  */
 952 VOID
 953 NTAPI
 954 KiCoprocessorError(VOID)
 955 {
 956     PFX_SAVE_AREA NpxArea;
 957
 958     /* Get the FPU area */
 959     NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
 960
 961     /* Set CR0_TS */
 962     NpxArea->Cr0NpxState = CR0_TS;
 963     __writecr0(__readcr0() | CR0_TS);
 964 }
 965
 966 /*
 967  * @implemented
 968  */
 969 NTSTATUS
 970 NTAPI
 971 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
 972 {
 973     PFNSAVE_FORMAT FpState;
 974     ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
 975     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 976
 977     /* check if we are doing software emulation */
 978     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
 979
 980     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
 981     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
 982
 983     *((PVOID *) Save) = FpState;
 984 #ifdef __GNUC__
 985     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
 986 #else
 987     __asm
 988     {
 989         fnsave [FpState]
 990     };
 991 #endif
 992
 993     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
 994     return STATUS_SUCCESS;
 995 }
 996
 997 /*
 998  * @implemented
 999  */
1000 NTSTATUS
1001 NTAPI
1002 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1003 {
1004     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1005     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1006     DPRINT1("%s is not really implemented\n", __FUNCTION__);
1007
1008 #ifdef __GNUC__
1009     asm volatile("fnclex\n\t");
1010     asm volatile("frstor %0\n\t" : "=m" (*FpState));
1011 #else
1012     __asm
1013     {
1014         fnclex
1015         frstor [FpState]
1016     };
1017 #endif
1018
1019     ExFreePool(FpState);
1020     return STATUS_SUCCESS;
1021 }
1022
1023 /*
1024  * @implemented
1025  */
1026 ULONG
1027 NTAPI
1028 KeGetRecommendedSharedDataAlignment(VOID)
1029 {
1030     /* Return the global variable */
1031     return KeLargestCacheLine;
1032 }
1033
1034 VOID
1035 NTAPI
1036 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1037                       IN PVOID Ignored1,
1038                       IN PVOID Ignored2,
1039                       IN PVOID Ignored3)
1040 {
1041     /* Signal this packet as done */
1042     KiIpiSignalPacketDone(PacketContext);
1043
1044     /* Flush the TB for the Current CPU */
1045     KeFlushCurrentTb();
1046 }
1047
1048 /*
1049  * @implemented
1050  */
1051 VOID
1052 NTAPI
1053 KeFlushEntireTb(IN BOOLEAN Invalid,
1054                 IN BOOLEAN AllProcessors)
1055 {
1056     KIRQL OldIrql;
1057 #ifdef CONFIG_SMP
1058     KAFFINITY TargetAffinity;
1059     PKPRCB Prcb = KeGetCurrentPrcb();
1060 #endif
1061
1062     /* Raise the IRQL for the TB Flush */
1063     OldIrql = KeRaiseIrqlToSynchLevel();
1064
1065 #ifdef CONFIG_SMP
1066     /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1067
1068     /* Get the current processor affinity, and exclude ourselves */
1069     TargetAffinity = KeActiveProcessors;
1070     TargetAffinity &= ~Prcb->SetMember;
1071
1072     /* Make sure this is MP */
1073     if (TargetAffinity)
1074     {
1075         /* Send an IPI TB flush to the other processors */
1076         KiIpiSendPacket(TargetAffinity,
1077                         KiFlushTargetEntireTb,
1078                         NULL,
1079                         0,
1080                         NULL);
1081     }
1082 #endif
1083
1084     /* Flush the TB for the Current CPU, and update the flush stamp */
1085     KeFlushCurrentTb();
1086
1087 #ifdef CONFIG_SMP
1088     /* If this is MP, wait for the other processors to finish */
1089     if (TargetAffinity)
1090     {
1091         /* Sanity check */
1092         ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1093
1094         /* FIXME: TODO */
1095         ASSERTMSG("Not yet implemented\n", FALSE);
1096     }
1097 #endif
1098
1099     /* Update the flush stamp and return to original IRQL */
1100     InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1101     KeLowerIrql(OldIrql);
1102 }
1103
1104 /*
1105  * @implemented
1106  */
1107 VOID
1108 NTAPI
1109 KeSetDmaIoCoherency(IN ULONG Coherency)
1110 {
1111     /* Save the coherency globally */
1112     KiDmaIoCoherency = Coherency;
1113 }
1114
1115 /*
1116  * @implemented
1117  */
1118 KAFFINITY
1119 NTAPI
1120 KeQueryActiveProcessors(VOID)
1121 {
1122     PAGED_CODE();
1123
1124     /* Simply return the number of active processors */
1125     return KeActiveProcessors;
1126 }
1127
1128 /*
1129  * @implemented
1130  */
1131 VOID
1132 __cdecl
1133 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1134 {
1135     /* Capture the context */
1136     RtlCaptureContext(&State->ContextFrame);
1137
1138     /* Capture the control state */
1139     KiSaveProcessorControlState(State);
1140 }