reactos/ntoskrnl/ke/i386/cpu.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/cpu.c
   5  * PURPOSE:         Routines for CPU-level support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* The Boot TSS */
  18 KTSS KiBootTss;
  19
  20 /* The TSS to use for Double Fault Traps (INT 0x9) */
  21 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
  22
  23 /* The TSS to use for NMI Fault Traps (INT 0x2) */
  24 UCHAR KiNMITSS[KTSS_IO_MAPS];
  25
  26 /* The Boot GDT */
  27 KGDTENTRY KiBootGdt[256] =
  28 {
  29     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_NULL */
  30     {0xffff, 0x0000, {{0x00, 0x9b, 0xcf, 0x00}}},       /* KGDT_R0_CODE */
  31     {0xffff, 0x0000, {{0x00, 0x93, 0xcf, 0x00}}},       /* KGDT_R0_DATA */
  32     {0xffff, 0x0000, {{0x00, 0xfb, 0xcf, 0x00}}},       /* KGDT_R3_CODE */
  33     {0xffff, 0x0000, {{0x00, 0xf3, 0xcf, 0x00}}},       /* KGDT_R3_DATA*/
  34     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_TSS */
  35     {0x0001, 0xf000, {{0xdf, 0x93, 0xc0, 0xff}}},       /* KGDT_R0_PCR */
  36     {0x0fff, 0x0000, {{0x00, 0xf3, 0x40, 0x00}}},       /* KGDT_R3_TEB */
  37     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_UNUSED */
  38     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_LDT */
  39     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}},       /* KGDT_DF_TSS */
  40     {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}        /* KGDT_NMI_TSS */
  41 };
  42
  43 /* GDT Descriptor */
  44 KDESCRIPTOR KiGdtDescriptor = {0, sizeof(KiBootGdt) - 1, (ULONG)KiBootGdt};
  45
  46 /* CPU Features and Flags */
  47 ULONG KeI386CpuType;
  48 ULONG KeI386CpuStep;
  49 ULONG KeProcessorArchitecture;
  50 ULONG KeProcessorLevel;
  51 ULONG KeProcessorRevision;
  52 ULONG KeFeatureBits;
  53 ULONG KiFastSystemCallDisable = 1;
  54 ULONG KeI386NpxPresent = 0;
  55 ULONG KiMXCsrMask = 0;
  56 ULONG MxcsrFeatureMask = 0;
  57 ULONG KeI386XMMIPresent = 0;
  58 ULONG KeI386FxsrPresent = 0;
  59 ULONG KeI386MachineType;
  60 ULONG Ke386Pae = FALSE;
  61 ULONG Ke386NoExecute = FALSE;
  62 ULONG KeLargestCacheLine = 0x40;
  63 ULONG KeDcacheFlushCount = 0;
  64 ULONG KeIcacheFlushCount = 0;
  65 ULONG KiDmaIoCoherency = 0;
  66 CHAR KeNumberProcessors;
  67 KAFFINITY KeActiveProcessors = 1;
  68 BOOLEAN KiI386PentiumLockErrataPresent;
  69 BOOLEAN KiSMTProcessorsPresent;
  70
  71 /* Freeze data */
  72 KIRQL KiOldIrql;
  73 ULONG KiFreezeFlag;
  74
  75 /* Flush data */
  76 volatile LONG KiTbFlushTimeStamp;
  77
  78 /* CPU Signatures */
  79 static const CHAR CmpIntelID[]       = "GenuineIntel";
  80 static const CHAR CmpAmdID[]         = "AuthenticAMD";
  81 static const CHAR CmpCyrixID[]       = "CyrixInstead";
  82 static const CHAR CmpTransmetaID[]   = "GenuineTMx86";
  83 static const CHAR CmpCentaurID[]     = "CentaurHauls";
  84 static const CHAR CmpRiseID[]        = "RiseRiseRise";
  85
  86 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
  87
  88 VOID
  89 NTAPI
  90 CPUID(OUT ULONG CpuInfo[4],
  91       IN ULONG InfoType)
  92 {
  93     Ki386Cpuid(InfoType, &CpuInfo[0], &CpuInfo[1], &CpuInfo[2], &CpuInfo[3]);
  94 }
  95
  96 VOID
  97 WRMSR(IN ULONG Register,
  98       IN LONGLONG Value)
  99 {
 100     LARGE_INTEGER LargeVal;
 101     LargeVal.QuadPart = Value;
 102     Ke386Wrmsr(Register, LargeVal.HighPart, LargeVal.LowPart);
 103 }
 104
 105 LONGLONG
 106 RDMSR(IN ULONG Register)
 107 {
 108     LARGE_INTEGER LargeVal = {{0}};
 109     Ke386Rdmsr(Register, LargeVal.HighPart, LargeVal.LowPart);
 110     return LargeVal.QuadPart;
 111 }
 112
 113 /* FUNCTIONS *****************************************************************/
 114
 115 VOID
 116 NTAPI
 117 KiSetProcessorType(VOID)
 118 {
 119     ULONG EFlags = 0, NewEFlags;
 120     ULONG Reg[4];
 121     ULONG Stepping, Type;
 122
 123     /* Start by assuming no CPUID data */
 124     KeGetCurrentPrcb()->CpuID = 0;
 125
 126     /* Save EFlags */
 127     Ke386SaveFlags(EFlags);
 128
 129     /* XOR out the ID bit and update EFlags */
 130     NewEFlags = EFlags ^ EFLAGS_ID;
 131     Ke386RestoreFlags(NewEFlags);
 132
 133     /* Get them back and see if they were modified */
 134     Ke386SaveFlags(NewEFlags);
 135     if (NewEFlags != EFlags)
 136     {
 137         /* The modification worked, so CPUID exists. Set the ID Bit again. */
 138         EFlags |= EFLAGS_ID;
 139         Ke386RestoreFlags(EFlags);
 140
 141         /* Peform CPUID 0 to see if CPUID 1 is supported */
 142         CPUID(Reg, 0);
 143         if (Reg[0] > 0)
 144         {
 145             /* Do CPUID 1 now */
 146             CPUID(Reg, 1);
 147
 148             /*
 149              * Get the Stepping and Type. The stepping contains both the
 150              * Model and the Step, while the Type contains the returned Type.
 151              * We ignore the family.
 152              *
 153              * For the stepping, we convert this: zzzzzzxy into this: x0y
 154              */
 155             Stepping = Reg[0] & 0xF0;
 156             Stepping <<= 4;
 157             Stepping += (Reg[0] & 0xFF);
 158             Stepping &= 0xF0F;
 159             Type = Reg[0] & 0xF00;
 160             Type >>= 8;
 161
 162             /* Save them in the PRCB */
 163             KeGetCurrentPrcb()->CpuID = TRUE;
 164             KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
 165             KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
 166         }
 167         else
 168         {
 169             DPRINT1("CPUID Support lacking\n");
 170         }
 171     }
 172     else
 173     {
 174         DPRINT1("CPUID Support lacking\n");
 175     }
 176
 177     /* Restore EFLAGS */
 178     Ke386RestoreFlags(EFlags);
 179 }
 180
 181 ULONG
 182 NTAPI
 183 KiGetCpuVendor(VOID)
 184 {
 185     PKPRCB Prcb = KeGetCurrentPrcb();
 186     ULONG Vendor[5];
 187     ULONG Temp;
 188
 189     /* Assume no Vendor ID and fail if no CPUID Support. */
 190     Prcb->VendorString[0] = 0;
 191     if (!Prcb->CpuID) return 0;
 192
 193     /* Get the Vendor ID and null-terminate it */
 194     CPUID(Vendor, 0);
 195     Vendor[4] = 0;
 196
 197     /* Re-arrange vendor string */
 198     Temp = Vendor[2];
 199     Vendor[2] = Vendor[3];
 200     Vendor[3] = Temp;
 201
 202     /* Copy it to the PRCB and null-terminate it again */
 203     RtlCopyMemory(Prcb->VendorString,
 204                   &Vendor[1],
 205                   sizeof(Prcb->VendorString) - sizeof(CHAR));
 206     Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
 207
 208     /* Now check the CPU Type */
 209     if (!strcmp(Prcb->VendorString, CmpIntelID))
 210     {
 211         return CPU_INTEL;
 212     }
 213     else if (!strcmp(Prcb->VendorString, CmpAmdID))
 214     {
 215         return CPU_AMD;
 216     }
 217     else if (!strcmp(Prcb->VendorString, CmpCyrixID))
 218     {
 219         DPRINT1("Cyrix CPUs not fully supported\n");
 220         return 0;
 221     }
 222     else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
 223     {
 224         DPRINT1("Transmeta CPUs not fully supported\n");
 225         return 0;
 226     }
 227     else if (!strcmp(Prcb->VendorString, CmpCentaurID))
 228     {
 229         DPRINT1("VIA CPUs not fully supported\n");
 230         return 0;
 231     }
 232     else if (!strcmp(Prcb->VendorString, CmpRiseID))
 233     {
 234         DPRINT1("Rise CPUs not fully supported\n");
 235         return 0;
 236     }
 237
 238     /* Invalid CPU */
 239     return 0;
 240 }
 241
 242 ULONG
 243 NTAPI
 244 KiGetFeatureBits(VOID)
 245 {
 246     PKPRCB Prcb = KeGetCurrentPrcb();
 247     ULONG Vendor;
 248     ULONG FeatureBits = KF_WORKING_PTE;
 249     ULONG Reg[4];
 250     BOOLEAN ExtendedCPUID = TRUE;
 251     ULONG CpuFeatures = 0;
 252
 253     /* Get the Vendor ID */
 254     Vendor = KiGetCpuVendor();
 255
 256     /* Make sure we got a valid vendor ID at least. */
 257     if (!Vendor) return FeatureBits;
 258
 259     /* Get the CPUID Info. Features are in Reg[3]. */
 260     CPUID(Reg, 1);
 261
 262     /* Set the initial APIC ID */
 263     Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
 264
 265     /* Check for AMD CPU */
 266     if (Vendor == CPU_AMD)
 267     {
 268         /* Check if this is a K5 or higher. */
 269         if ((Reg[0] & 0x0F00) >= 0x0500)
 270         {
 271             /* Check if this is a K5 specifically. */
 272             if ((Reg[0] & 0x0F00) == 0x0500)
 273             {
 274                 /* Get the Model Number */
 275                 switch (Reg[0] & 0x00F0)
 276                 {
 277                     /* Check if this is the Model 1 */
 278                     case 0x0010:
 279
 280                         /* Check if this is Step 0 or 1. They don't support PGE */
 281                         if ((Reg[0] & 0x000F) > 0x03) break;
 282
 283                     case 0x0000:
 284
 285                         /* Model 0 doesn't support PGE at all. */
 286                         Reg[3] &= ~0x2000;
 287                         break;
 288
 289                     case 0x0080:
 290
 291                         /* K6-2, Step 8 and over have support for MTRR. */
 292                         if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
 293                         break;
 294
 295                     case 0x0090:
 296
 297                         /* As does the K6-3 */
 298                         FeatureBits |= KF_AMDK6MTRR;
 299                         break;
 300
 301                     default:
 302                         break;
 303                 }
 304             }
 305         }
 306         else
 307         {
 308             /* Families below 5 don't support PGE, PSE or CMOV at all */
 309             Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
 310
 311             /* They also don't support advanced CPUID functions. */
 312             ExtendedCPUID = FALSE;
 313         }
 314
 315         /* Set the current features */
 316         CpuFeatures = Reg[3];
 317     }
 318
 319     /* Now check if this is Intel */
 320     if (Vendor == CPU_INTEL)
 321     {
 322         /* Check if it's a P6 */
 323         if (Prcb->CpuType == 6)
 324         {
 325             /* Perform the special sequence to get the MicroCode Signature */
 326             WRMSR(0x8B, 0);
 327             CPUID(Reg, 1);
 328             Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
 329         }
 330         else if (Prcb->CpuType == 5)
 331         {
 332             /* On P5, enable workaround for the LOCK errata. */
 333             KiI386PentiumLockErrataPresent = TRUE;
 334         }
 335
 336         /* Check for broken P6 with bad SMP PTE implementation */
 337         if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
 338             ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
 339         {
 340             /* Remove support for correct PTE support. */
 341             FeatureBits &= ~KF_WORKING_PTE;
 342         }
 343
 344         /* Check if the CPU is too old to support SYSENTER */
 345         if ((Prcb->CpuType < 6) ||
 346             ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
 347         {
 348             /* Disable it */
 349             Reg[3] &= ~0x800;
 350         }
 351
 352         /* Set the current features */
 353         CpuFeatures = Reg[3];
 354     }
 355
 356 #ifdef CPU_TRANSMETA
 357         if (Vendor == CPU_TRANSMETA)
 358         {
 359                 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
 360                 if ((Reg[0] & 0x0F00) >= 0x0500 && (Reg[0] & 0x00FF) >= 0x0042)
 361                 {
 362                         WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
 363                         FeatureBits |= KF_CMPXCHG8B;
 364                 }
 365         }
 366 #endif
 367
 368 #ifdef CPU_CENTAUR
 369         if (Vendor == CPU_CENTAUR)
 370         {
 371                 /* If CMPXCHG8B is not detected, try to enable it */
 372                 if (!(CpuFeatures & 0x00000100))
 373                 {
 374                         if ((Reg[0] & 0x0F00) == 0x0500)
 375                         {
 376                                 WRMSR(0x0107, RDMSR(0x0107) | 0x02);
 377                                 FeatureBits |= KF_CMPXCHG8B;
 378                         }
 379                         else if ((Reg[0] & 0x0F00) >= 0x0600)
 380                         {
 381                                 WRMSR(0x1107, (RDMSR(0x1107) | 0x02) & ~((LONGLONG)0x01));
 382                                 FeatureBits |= KF_CMPXCHG8B;
 383                         }
 384                 }
 385         }
 386 #endif
 387
 388 #ifdef CPU_RISE
 389         if (Vendor == CPU_RISE)
 390         {
 391                 /* Windows Vista assumes CMPXCHG8B is always supported on Rise */
 392                 FeatureBits |= KF_CMPXCHG8B;
 393         }
 394 #endif
 395
 396     /* Convert all CPUID Feature bits into our format */
 397     if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
 398     if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
 399     if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
 400     if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
 401     if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
 402     if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
 403     if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
 404     if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
 405     if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
 406     if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
 407     if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
 408     if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
 409     if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
 410     if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
 411
 412     /* Check if the CPU has hyper-threading */
 413     if (CpuFeatures & 0x10000000)
 414     {
 415         /* Set the number of logical CPUs */
 416         Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
 417         if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
 418         {
 419             /* We're on dual-core */
 420             KiSMTProcessorsPresent = TRUE;
 421         }
 422     }
 423     else
 424     {
 425         /* We only have a single CPU */
 426         Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
 427     }
 428
 429     /* Check if CPUID 0x80000000 is supported */
 430     if (ExtendedCPUID)
 431     {
 432         /* Do the call */
 433         CPUID(Reg, 0x80000000);
 434         if ((Reg[0] & 0xffffff00) == 0x80000000)
 435         {
 436             /* Check if CPUID 0x80000001 is supported */
 437             if (Reg[0] >= 0x80000001)
 438             {
 439                 /* Check which extended features are available. */
 440                 CPUID(Reg, 0x80000001);
 441
 442                 /* Check if NX-bit is supported */
 443                 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
 444
 445                 /* Now handle each features for each CPU Vendor */
 446                 switch (Vendor)
 447                 {
 448                     case CPU_AMD:
 449                         if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
 450                         break;
 451                 }
 452             }
 453         }
 454     }
 455
 456     /* Return the Feature Bits */
 457     return FeatureBits;
 458 }
 459
 460 VOID
 461 NTAPI
 462 KiGetCacheInformation(VOID)
 463 {
 464     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 465     ULONG Vendor;
 466     ULONG Data[4];
 467     ULONG CacheRequests = 0, i;
 468     ULONG CurrentRegister;
 469     UCHAR RegisterByte;
 470     BOOLEAN FirstPass = TRUE;
 471
 472     /* Set default L2 size */
 473     Pcr->SecondLevelCacheSize = 0;
 474
 475     /* Get the Vendor ID and make sure we support CPUID */
 476     Vendor = KiGetCpuVendor();
 477     if (!Vendor) return;
 478
 479     /* Check the Vendor ID */
 480     switch (Vendor)
 481     {
 482         /* Handle Intel case */
 483         case CPU_INTEL:
 484
 485             /*Check if we support CPUID 2 */
 486             CPUID(Data, 0);
 487             if (Data[0] >= 2)
 488             {
 489                 /* We need to loop for the number of times CPUID will tell us to */
 490                 do
 491                 {
 492                     /* Do the CPUID call */
 493                     CPUID(Data, 2);
 494
 495                     /* Check if it was the first call */
 496                     if (FirstPass)
 497                     {
 498                         /*
 499                          * The number of times to loop is the first byte. Read
 500                          * it and then destroy it so we don't get confused.
 501                          */
 502                         CacheRequests = Data[0] & 0xFF;
 503                         Data[0] &= 0xFFFFFF00;
 504
 505                         /* Don't go over this again */
 506                         FirstPass = FALSE;
 507                     }
 508
 509                     /* Loop all 4 registers */
 510                     for (i = 0; i < 4; i++)
 511                     {
 512                         /* Get the current register */
 513                         CurrentRegister = Data[i];
 514
 515                         /*
 516                          * If the upper bit is set, then this register should
 517                          * be skipped.
 518                          */
 519                         if (CurrentRegister & 0x80000000) continue;
 520
 521                         /* Keep looping for every byte inside this register */
 522                         while (CurrentRegister)
 523                         {
 524                             /* Read a byte, skip a byte. */
 525                             RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
 526                             CurrentRegister >>= 8;
 527                             if (!RegisterByte) continue;
 528
 529                             /*
 530                              * Valid values are from 0x40 (0 bytes) to 0x49
 531                              * (32MB), or from 0x80 to 0x89 (same size but
 532                              * 8-way associative.
 533                              */
 534                             if (((RegisterByte > 0x40) &&
 535                                  (RegisterByte <= 0x49)) ||
 536                                 ((RegisterByte > 0x80) &&
 537                                 (RegisterByte <= 0x89)))
 538                             {
 539                                 /* Mask out only the first nibble */
 540                                 RegisterByte &= 0x0F;
 541
 542                                 /* Set the L2 Cache Size */
 543                                 Pcr->SecondLevelCacheSize = 0x10000 <<
 544                                                             RegisterByte;
 545                             }
 546                         }
 547                     }
 548                 } while (--CacheRequests);
 549             }
 550             break;
 551
 552         case CPU_AMD:
 553
 554             /* Check if we support CPUID 0x80000006 */
 555             CPUID(Data, 0x80000000);
 556             if (Data[0] >= 6)
 557             {
 558                 /* Get 2nd level cache and tlb size */
 559                 CPUID(Data, 0x80000006);
 560
 561                 /* Set the L2 Cache Size */
 562                 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
 563             }
 564             break;
 565     }
 566 }
 567
 568 VOID
 569 NTAPI
 570 KiSetCR0Bits(VOID)
 571 {
 572     ULONG Cr0;
 573
 574     /* Save current CR0 */
 575     Cr0 = __readcr0();
 576
 577     /* If this is a 486, enable Write-Protection */
 578     if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
 579
 580     /* Set new Cr0 */
 581     __writecr0(Cr0);
 582 }
 583
 584 VOID
 585 NTAPI
 586 KiInitializeTSS2(IN PKTSS Tss,
 587                  IN PKGDTENTRY TssEntry OPTIONAL)
 588 {
 589     PUCHAR p;
 590
 591     /* Make sure the GDT Entry is valid */
 592     if (TssEntry)
 593     {
 594         /* Set the Limit */
 595         TssEntry->LimitLow = sizeof(KTSS) - 1;
 596         TssEntry->HighWord.Bits.LimitHi = 0;
 597     }
 598
 599     /* Now clear the I/O Map */
 600     RtlFillMemory(Tss->IoMaps[0].IoMap, 8096, -1);
 601
 602     /* Initialize Interrupt Direction Maps */
 603     p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
 604     RtlZeroMemory(p, 32);
 605
 606     /* Add DPMI support for interrupts */
 607     p[0] = 4;
 608     p[3] = 0x18;
 609     p[4] = 0x18;
 610
 611     /* Initialize the default Interrupt Direction Map */
 612     p = Tss->IntDirectionMap;
 613     RtlZeroMemory(Tss->IntDirectionMap, 32);
 614
 615     /* Add DPMI support */
 616     p[0] = 4;
 617     p[3] = 0x18;
 618     p[4] = 0x18;
 619 }
 620
 621 VOID
 622 NTAPI
 623 KiInitializeTSS(IN PKTSS Tss)
 624 {
 625     /* Set an invalid map base */
 626     Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
 627
 628     /* Disable traps during Task Switches */
 629     Tss->Flags = 0;
 630
 631     /* Set LDT and Ring 0 SS */
 632     Tss->LDT = 0;
 633     Tss->Ss0 = KGDT_R0_DATA;
 634 }
 635
 636 VOID
 637 FASTCALL
 638 Ki386InitializeTss(IN PKTSS Tss,
 639                    IN PKIDTENTRY Idt,
 640                    IN PKGDTENTRY Gdt)
 641 {
 642     PKGDTENTRY TssEntry, TaskGateEntry;
 643
 644     /* Initialize the boot TSS. */
 645     TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
 646     TssEntry->HighWord.Bits.Type = I386_TSS;
 647     TssEntry->HighWord.Bits.Pres = 1;
 648     TssEntry->HighWord.Bits.Dpl = 0;
 649     KiInitializeTSS2(Tss, TssEntry);
 650     KiInitializeTSS(Tss);
 651
 652     /* Load the task register */
 653     Ke386SetTr(KGDT_TSS);
 654
 655     /* Setup the Task Gate for Double Fault Traps */
 656     TaskGateEntry = (PKGDTENTRY)&Idt[8];
 657     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 658     TaskGateEntry->HighWord.Bits.Pres = 1;
 659     TaskGateEntry->HighWord.Bits.Dpl = 0;
 660     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
 661
 662     /* Initialize the TSS used for handling double faults. */
 663     Tss = (PKTSS)KiDoubleFaultTSS;
 664     KiInitializeTSS(Tss);
 665     Tss->CR3 = __readcr3();
 666     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 667     Tss->Eip = PtrToUlong(KiTrap8);
 668     Tss->Cs = KGDT_R0_CODE;
 669     Tss->Fs = KGDT_R0_PCR;
 670     Tss->Ss = Ke386GetSs();
 671     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 672     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 673
 674     /* Setup the Double Trap TSS entry in the GDT */
 675     TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
 676     TssEntry->HighWord.Bits.Type = I386_TSS;
 677     TssEntry->HighWord.Bits.Pres = 1;
 678     TssEntry->HighWord.Bits.Dpl = 0;
 679     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 680     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 681     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 682     TssEntry->LimitLow = KTSS_IO_MAPS;
 683
 684     /* Now setup the NMI Task Gate */
 685     TaskGateEntry = (PKGDTENTRY)&Idt[2];
 686     TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
 687     TaskGateEntry->HighWord.Bits.Pres = 1;
 688     TaskGateEntry->HighWord.Bits.Dpl = 0;
 689     ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
 690
 691     /* Initialize the actual TSS */
 692     Tss = (PKTSS)KiNMITSS;
 693     KiInitializeTSS(Tss);
 694     Tss->CR3 = __readcr3();
 695     Tss->Esp0 = PtrToUlong(KiDoubleFaultStack);
 696     Tss->Eip = PtrToUlong(KiTrap2);
 697     Tss->Cs = KGDT_R0_CODE;
 698     Tss->Fs = KGDT_R0_PCR;
 699     Tss->Ss = Ke386GetSs();
 700     Tss->Es = KGDT_R3_DATA | RPL_MASK;
 701     Tss->Ds = KGDT_R3_DATA | RPL_MASK;
 702
 703     /* And its associated TSS Entry */
 704     TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
 705     TssEntry->HighWord.Bits.Type = I386_TSS;
 706     TssEntry->HighWord.Bits.Pres = 1;
 707     TssEntry->HighWord.Bits.Dpl = 0;
 708     TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
 709     TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
 710     TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
 711     TssEntry->LimitLow = KTSS_IO_MAPS;
 712 }
 713
 714 VOID
 715 NTAPI
 716 KeFlushCurrentTb(VOID)
 717 {
 718     /* Flush the TLB by resetting CR3 */
 719     __writecr3(__readcr3());
 720 }
 721
 722 VOID
 723 NTAPI
 724 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
 725 {
 726     /* Restore the CR registers */
 727     __writecr0(ProcessorState->SpecialRegisters.Cr0);
 728     Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
 729     __writecr3(ProcessorState->SpecialRegisters.Cr3);
 730     if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
 731
 732     //
 733     // Restore the DR registers
 734     //
 735     Ke386SetDr0(ProcessorState->SpecialRegisters.KernelDr0);
 736     Ke386SetDr1(ProcessorState->SpecialRegisters.KernelDr1);
 737     Ke386SetDr2(ProcessorState->SpecialRegisters.KernelDr2);
 738     Ke386SetDr3(ProcessorState->SpecialRegisters.KernelDr3);
 739     Ke386SetDr6(ProcessorState->SpecialRegisters.KernelDr6);
 740     Ke386SetDr7(ProcessorState->SpecialRegisters.KernelDr7);
 741
 742     //
 743     // Restore GDT, IDT, LDT and TSS
 744     //
 745     Ke386SetGlobalDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Gdtr.Limit);
 746     Ke386SetInterruptDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Idtr.Limit);
 747     Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
 748     Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 749 }
 750
 751 VOID
 752 NTAPI
 753 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
 754 {
 755     /* Save the CR registers */
 756     ProcessorState->SpecialRegisters.Cr0 = __readcr0();
 757     ProcessorState->SpecialRegisters.Cr2 = __readcr2();
 758     ProcessorState->SpecialRegisters.Cr3 = __readcr3();
 759     ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
 760                                            __readcr4() : 0;
 761
 762     /* Save the DR registers */
 763     ProcessorState->SpecialRegisters.KernelDr0 = Ke386GetDr0();
 764     ProcessorState->SpecialRegisters.KernelDr1 = Ke386GetDr1();
 765     ProcessorState->SpecialRegisters.KernelDr2 = Ke386GetDr2();
 766     ProcessorState->SpecialRegisters.KernelDr3 = Ke386GetDr3();
 767     ProcessorState->SpecialRegisters.KernelDr6 = Ke386GetDr6();
 768     ProcessorState->SpecialRegisters.KernelDr7 = Ke386GetDr7();
 769     Ke386SetDr7(0);
 770
 771     /* Save GDT, IDT, LDT and TSS */
 772     Ke386GetGlobalDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Gdtr.Limit);
 773     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&ProcessorState->SpecialRegisters.Idtr.Limit);
 774     Ke386GetTr(ProcessorState->SpecialRegisters.Tr);
 775     Ke386GetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
 776 }
 777
 778 VOID
 779 NTAPI
 780 KiInitializeMachineType(VOID)
 781 {
 782     /* Set the Machine Type we got from NTLDR */
 783     KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
 784 }
 785
 786 ULONG_PTR
 787 NTAPI
 788 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
 789 {
 790     /* Set CS and ESP */
 791     Ke386Wrmsr(0x174, KGDT_R0_CODE, 0);
 792     Ke386Wrmsr(0x175, (ULONG)KeGetCurrentPrcb()->DpcStack, 0);
 793
 794     /* Set LSTAR */
 795     Ke386Wrmsr(0x176, (ULONG)KiFastCallEntry, 0);
 796     return 0;
 797 }
 798
 799 VOID
 800 NTAPI
 801 KiRestoreFastSyscallReturnState(VOID)
 802 {
 803     /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
 804
 805     /* Check if the CPU Supports fast system call */
 806     if (KeFeatureBits & KF_FAST_SYSCALL)
 807     {
 808         /* Do an IPI to enable it */
 809         KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
 810     }
 811 }
 812
 813 ULONG_PTR
 814 NTAPI
 815 Ki386EnableDE(IN ULONG_PTR Context)
 816 {
 817     /* Enable DE */
 818     __writecr4(__readcr4() | CR4_DE);
 819     return 0;
 820 }
 821
 822 ULONG_PTR
 823 NTAPI
 824 Ki386EnableFxsr(IN ULONG_PTR Context)
 825 {
 826     /* Enable FXSR */
 827     __writecr4(__readcr4() | CR4_FXSR);
 828     return 0;
 829 }
 830
 831 ULONG_PTR
 832 NTAPI
 833 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
 834 {
 835     PKIDTENTRY IdtEntry;
 836
 837     /* Get the IDT Entry for Interrupt 19 */
 838     IdtEntry = &((PKIPCR)KeGetPcr())->IDT[19];
 839
 840     /* Set it up */
 841     IdtEntry->Selector = KGDT_R0_CODE;
 842     IdtEntry->Offset = ((ULONG_PTR)KiTrap19 & 0xFFFF);
 843     IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap19 >> 16) & 0xFFFF;
 844     ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
 845     ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
 846     ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
 847
 848     /* Enable XMMI exceptions */
 849     __writecr4(__readcr4() | CR4_XMMEXCPT);
 850     return 0;
 851 }
 852
 853 VOID
 854 NTAPI
 855 KiI386PentiumLockErrataFixup(VOID)
 856 {
 857     KDESCRIPTOR IdtDescriptor = {0};
 858     PKIDTENTRY NewIdt, NewIdt2;
 859
 860     /* Allocate memory for a new IDT */
 861     NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
 862
 863     /* Put everything after the first 7 entries on a new page */
 864     NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
 865
 866     /* Disable interrupts */
 867     _disable();
 868
 869     /* Get the current IDT and copy it */
 870     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 871     RtlCopyMemory(NewIdt2,
 872                   (PVOID)IdtDescriptor.Base,
 873                   IdtDescriptor.Limit + 1);
 874     IdtDescriptor.Base = (ULONG)NewIdt2;
 875
 876     /* Set the new IDT */
 877     Ke386SetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 878     ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
 879
 880     /* Restore interrupts */
 881     _enable();
 882
 883     /* Set the first 7 entries as read-only to produce a fault */
 884     MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
 885 }
 886
 887 BOOLEAN
 888 NTAPI
 889 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
 890                   IN PKEXCEPTION_FRAME ExceptionFrame)
 891 {
 892     ULONG Flags = 0;
 893
 894     /* Disable interrupts and get previous state */
 895     Ke386SaveFlags(Flags);
 896     //Flags = __getcallerseflags();
 897     _disable();
 898
 899     /* Save freeze flag */
 900     KiFreezeFlag = 4;
 901
 902     /* Save the old IRQL */
 903     KiOldIrql = KeGetCurrentIrql();
 904
 905     /* Return whether interrupts were enabled */
 906     return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 907 }
 908
 909 VOID
 910 NTAPI
 911 KeThawExecution(IN BOOLEAN Enable)
 912 {
 913     /* Cleanup CPU caches */
 914     KeFlushCurrentTb();
 915
 916     /* Re-enable interrupts */
 917     if (Enable) _enable();
 918 }
 919
 920 BOOLEAN
 921 NTAPI
 922 KeInvalidateAllCaches(VOID)
 923 {
 924     /* Only supported on Pentium Pro and higher */
 925     if (KeI386CpuType < 6) return FALSE;
 926
 927     /* Invalidate all caches */
 928     __wbinvd();
 929     return TRUE;
 930 }
 931
 932 VOID
 933 FASTCALL
 934 KeZeroPages(IN PVOID Address,
 935             IN ULONG Size)
 936 {
 937     /* Not using XMMI in this routine */
 938     RtlZeroMemory(Address, Size);
 939 }
 940
 941 /* PUBLIC FUNCTIONS **********************************************************/
 942
 943 /*
 944  * @implemented
 945  */
 946 NTSTATUS
 947 NTAPI
 948 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
 949 {
 950     PFNSAVE_FORMAT FpState;
 951     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 952     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 953
 954     /* check if we are doing software emulation */
 955     if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
 956
 957     FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
 958     if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
 959
 960     *((PVOID *) Save) = FpState;
 961 #ifdef __GNUC__
 962     asm volatile("fnsave %0\n\t" : "=m" (*FpState));
 963 #else
 964     __asm
 965     {
 966         fnsave [FpState]
 967     };
 968 #endif
 969
 970     KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
 971     return STATUS_SUCCESS;
 972 }
 973
 974 /*
 975  * @implemented
 976  */
 977 NTSTATUS
 978 NTAPI
 979 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
 980 {
 981     PFNSAVE_FORMAT FpState = *((PVOID *) Save);
 982     ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
 983     DPRINT1("%s is not really implemented\n", __FUNCTION__);
 984
 985 #ifdef __GNUC__
 986     asm volatile("fnclex\n\t");
 987     asm volatile("frstor %0\n\t" : "=m" (*FpState));
 988 #else
 989     __asm
 990     {
 991         fnclex
 992         frstor [FpState]
 993     };
 994 #endif
 995
 996     ExFreePool(FpState);
 997     return STATUS_SUCCESS;
 998 }
 999
1000 /*
1001  * @implemented
1002  */
1003 ULONG
1004 NTAPI
1005 KeGetRecommendedSharedDataAlignment(VOID)
1006 {
1007     /* Return the global variable */
1008     return KeLargestCacheLine;
1009 }
1010
1011 /*
1012  * @implemented
1013  */
1014 VOID
1015 NTAPI
1016 KeFlushEntireTb(IN BOOLEAN Invalid,
1017                 IN BOOLEAN AllProcessors)
1018 {
1019     KIRQL OldIrql;
1020
1021     /* Raise the IRQL for the TB Flush */
1022     OldIrql = KeRaiseIrqlToSynchLevel();
1023
1024 #ifdef CONFIG_SMP
1025     /* FIXME: Support IPI Flush */
1026 #error Not yet implemented!
1027 #endif
1028
1029     /* Flush the TB for the Current CPU */
1030     KeFlushCurrentTb();
1031
1032     /* Return to Original IRQL */
1033     KeLowerIrql(OldIrql);
1034 }
1035
1036 /*
1037  * @implemented
1038  */
1039 VOID
1040 NTAPI
1041 KeSetDmaIoCoherency(IN ULONG Coherency)
1042 {
1043     /* Save the coherency globally */
1044     KiDmaIoCoherency = Coherency;
1045 }
1046
1047 /*
1048  * @implemented
1049  */
1050 KAFFINITY
1051 NTAPI
1052 KeQueryActiveProcessors(VOID)
1053 {
1054     PAGED_CODE();
1055
1056     /* Simply return the number of active processors */
1057     return KeActiveProcessors;
1058 }
1059
1060 /*
1061  * @implemented
1062  */
1063 VOID
1064 __cdecl
1065 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1066 {
1067     /* Capture the context */
1068     RtlCaptureContext(&State->ContextFrame);
1069
1070     /* Capture the control state */
1071     KiSaveProcessorControlState(State);
1072 }