ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/i386/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 INIT_SECTION
  35 VOID
  36 NTAPI
  37 KiInitMachineDependent(VOID)
  38 {
  39     ULONG CpuCount;
  40     BOOLEAN FbCaching = FALSE;
  41     NTSTATUS Status;
  42     ULONG ReturnLength;
  43     ULONG i, Affinity, Sample = 0;
  44     PFX_SAVE_AREA FxSaveArea;
  45     ULONG MXCsrMask = 0xFFBF;
  46     CPU_INFO CpuInfo;
  47     KI_SAMPLE_MAP Samples[10];
  48     PKI_SAMPLE_MAP CurrentSample = Samples;
  49     LARGE_IDENTITY_MAP IdentityMap;
  50
  51     /* Check for large page support */
  52     if (KeFeatureBits & KF_LARGE_PAGE)
  53     {
  54         /* Do an IPI to enable it on all CPUs */
  55         if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
  56             KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
  57
  58         /* Free the pages allocated for identity map */
  59         Ki386FreeIdentityMap(&IdentityMap);
  60     }
  61
  62     /* Check for global page support */
  63     if (KeFeatureBits & KF_GLOBAL_PAGE)
  64     {
  65         /* Do an IPI to enable it on all CPUs */
  66         CpuCount = KeNumberProcessors;
  67         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  68     }
  69
  70     /* Check for PAT and/or MTRR support */
  71     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  72     {
  73         /* Query the HAL to make sure we can use it */
  74         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  75                                            sizeof(BOOLEAN),
  76                                            &FbCaching,
  77                                            &ReturnLength);
  78         if ((NT_SUCCESS(Status)) && (FbCaching))
  79         {
  80             /* We can't, disable it */
  81             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  82         }
  83     }
  84
  85     /* Check for PAT support and enable it */
  86     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  87
  88     /* Assume no errata for now */
  89     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  90
  91     /* Check if we have an NPX */
  92     if (KeI386NpxPresent)
  93     {
  94         /* Loop every CPU */
  95         i = KeActiveProcessors;
  96         for (Affinity = 1; i; Affinity <<= 1)
  97         {
  98             /* Check if this is part of the set */
  99             if (i & Affinity)
 100             {
 101                 /* Run on this CPU */
 102                 i &= ~Affinity;
 103                 KeSetSystemAffinityThread(Affinity);
 104
 105                 /* Detect FPU errata */
 106                 if (KiIsNpxErrataPresent())
 107                 {
 108                     /* Disable NPX support */
 109                     KeI386NpxPresent = FALSE;
 110                     SharedUserData->
 111                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 112                         TRUE;
 113                     break;
 114                 }
 115             }
 116         }
 117     }
 118
 119     /* If there's no NPX, then we're emulating the FPU */
 120     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 121         !KeI386NpxPresent;
 122
 123     /* Check if there's no NPX, so that we can disable associated features */
 124     if (!KeI386NpxPresent)
 125     {
 126         /* Remove NPX-related bits */
 127         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 128
 129         /* Disable kernel flags */
 130         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 131
 132         /* Disable processor features that might've been set until now */
 133         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 134         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 135         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 136         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 137         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 138     }
 139
 140     /* Check for CR4 support */
 141     if (KeFeatureBits & KF_CR4)
 142     {
 143         /* Do an IPI call to enable the Debug Exceptions */
 144         CpuCount = KeNumberProcessors;
 145         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 146     }
 147
 148     /* Check if FXSR was found */
 149     if (KeFeatureBits & KF_FXSR)
 150     {
 151         /* Do an IPI call to enable the FXSR */
 152         CpuCount = KeNumberProcessors;
 153         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 154
 155         /* Check if XMM was found too */
 156         if (KeFeatureBits & KF_XMMI)
 157         {
 158             /* Do an IPI call to enable XMMI exceptions */
 159             CpuCount = KeNumberProcessors;
 160             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 161
 162             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 163
 164             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 165             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP
 166         }
 167     }
 168
 169     /* Check for, and enable SYSENTER support */
 170     KiRestoreFastSyscallReturnState();
 171
 172     /* Loop every CPU */
 173     i = KeActiveProcessors;
 174     for (Affinity = 1; i; Affinity <<= 1)
 175     {
 176         /* Check if this is part of the set */
 177         if (i & Affinity)
 178         {
 179             /* Run on this CPU */
 180             i &= ~Affinity;
 181             KeSetSystemAffinityThread(Affinity);
 182
 183             /* Reset MHz to 0 for this CPU */
 184             KeGetCurrentPrcb()->MHz = 0;
 185
 186             /* Check if we can use RDTSC */
 187             if (KeFeatureBits & KF_RDTSC)
 188             {
 189                 /* Start sampling loop */
 190                 for (;;)
 191                 {
 192                     /* Do a dummy CPUID to start the sample */
 193                     KiCpuId(&CpuInfo, 0);
 194
 195                     /* Fill out the starting data */
 196                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 197                     CurrentSample->TSCStart = __rdtsc();
 198                     CurrentSample->PerfFreq.QuadPart = -50000;
 199
 200                     /* Sleep for this sample */
 201                     KeDelayExecutionThread(KernelMode,
 202                                            FALSE,
 203                                            &CurrentSample->PerfFreq);
 204
 205                     /* Do another dummy CPUID */
 206                     KiCpuId(&CpuInfo, 0);
 207
 208                     /* Fill out the ending data */
 209                     CurrentSample->PerfEnd =
 210                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 211                     CurrentSample->TSCEnd = __rdtsc();
 212
 213                     /* Calculate the differences */
 214                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 215                                                CurrentSample->PerfStart.QuadPart;
 216                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 217                                               CurrentSample->TSCStart;
 218
 219                     /* Compute CPU Speed */
 220                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 221                                                   CurrentSample->
 222                                                   PerfFreq.QuadPart + 500000) /
 223                                                  (CurrentSample->PerfDelta *
 224                                                   1000000));
 225
 226                     /* Check if this isn't the first sample */
 227                     if (Sample)
 228                     {
 229                         /* Check if we got a good precision within 1MHz */
 230                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 231                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 232                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 233                         {
 234                             /* We did, stop sampling */
 235                             break;
 236                         }
 237                     }
 238
 239                     /* Move on */
 240                     CurrentSample++;
 241                     Sample++;
 242
 243                     if (Sample == RTL_NUMBER_OF(Samples))
 244                     {
 245                         /* No luck. Average the samples and be done */
 246                         ULONG TotalMHz = 0;
 247                         while (Sample--)
 248                         {
 249                             TotalMHz += Samples[Sample].MHz;
 250                         }
 251                         CurrentSample[-1].MHz = TotalMHz / RTL_NUMBER_OF(Samples);
 252                         DPRINT1("Sampling CPU frequency failed. Using average of %lu MHz\n", CurrentSample[-1].MHz);
 253                         break;
 254                     }
 255                 }
 256
 257                 /* Save the CPU Speed */
 258                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 259             }
 260
 261             /* Check if we have MTRR */
 262             if (KeFeatureBits & KF_MTRR)
 263             {
 264                 /* Then manually initialize MTRR for the CPU */
 265                 KiInitializeMTRR(i ? FALSE : TRUE);
 266             }
 267
 268             /* Check if we have AMD MTRR and initialize it for the CPU */
 269             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 270
 271             /* Check if this is a buggy Pentium and apply the fixup if so */
 272             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 273
 274             /* Check if the CPU supports FXSR */
 275             if (KeFeatureBits & KF_FXSR)
 276             {
 277                 /* Get the current thread NPX state */
 278                 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
 279
 280                 /* Clear initial MXCsr mask */
 281                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 282
 283                 /* Save the current NPX State */
 284                 Ke386SaveFpuState(FxSaveArea);
 285
 286                 /* Check if the current mask doesn't match the reserved bits */
 287                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 288                 {
 289                     /* Then use whatever it's holding */
 290                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 291                 }
 292
 293                 /* Check if nobody set the kernel-wide mask */
 294                 if (!KiMXCsrMask)
 295                 {
 296                     /* Then use the one we calculated above */
 297                     KiMXCsrMask = MXCsrMask;
 298                 }
 299                 else
 300                 {
 301                     /* Was it set to the same value we found now? */
 302                     if (KiMXCsrMask != MXCsrMask)
 303                     {
 304                         /* No, something is definitely wrong */
 305                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 306                                      KF_FXSR,
 307                                      KiMXCsrMask,
 308                                      MXCsrMask,
 309                                      0);
 310                     }
 311                 }
 312
 313                 /* Now set the kernel mask */
 314                 KiMXCsrMask &= MXCsrMask;
 315             }
 316         }
 317     }
 318
 319     /* Return affinity back to where it was */
 320     KeRevertToUserAffinityThread();
 321
 322     /* NT allows limiting the duration of an ISR with a registry key */
 323     if (KiTimeLimitIsrMicroseconds)
 324     {
 325         /* FIXME: TODO */
 326         DPRINT1("ISR Time Limit not yet supported\n");
 327     }
 328
 329     /* Set CR0 features based on detected CPU */
 330     KiSetCR0Bits();
 331 }
 332
 333 INIT_SECTION
 334 VOID
 335 NTAPI
 336 KiInitializePcr(IN ULONG ProcessorNumber,
 337                 IN PKIPCR Pcr,
 338                 IN PKIDTENTRY Idt,
 339                 IN PKGDTENTRY Gdt,
 340                 IN PKTSS Tss,
 341                 IN PKTHREAD IdleThread,
 342                 IN PVOID DpcStack)
 343 {
 344     /* Setup the TIB */
 345     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 346     Pcr->NtTib.StackBase = 0;
 347     Pcr->NtTib.StackLimit = 0;
 348     Pcr->NtTib.Self = NULL;
 349
 350     /* Set the Current Thread */
 351     Pcr->PrcbData.CurrentThread = IdleThread;
 352
 353     /* Set pointers to ourselves */
 354     Pcr->SelfPcr = (PKPCR)Pcr;
 355     Pcr->Prcb = &Pcr->PrcbData;
 356
 357     /* Set the PCR Version */
 358     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 359     Pcr->MinorVersion = PCR_MINOR_VERSION;
 360
 361     /* Set the PCRB Version */
 362     Pcr->PrcbData.MajorVersion = 1;
 363     Pcr->PrcbData.MinorVersion = 1;
 364
 365     /* Set the Build Type */
 366     Pcr->PrcbData.BuildType = 0;
 367 #ifndef CONFIG_SMP
 368     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 369 #endif
 370 #if DBG
 371     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 372 #endif
 373
 374     /* Set the Processor Number and current Processor Mask */
 375     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 376     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 377
 378     /* Set the PRCB for this Processor */
 379     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 380
 381     /* Start us out at PASSIVE_LEVEL */
 382     Pcr->Irql = PASSIVE_LEVEL;
 383
 384     /* Set the GDI, IDT, TSS and DPC Stack */
 385     Pcr->GDT = (PVOID)Gdt;
 386     Pcr->IDT = Idt;
 387     Pcr->TSS = Tss;
 388     Pcr->TssCopy = Tss;
 389     Pcr->PrcbData.DpcStack = DpcStack;
 390
 391     /* Setup the processor set */
 392     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 393 }
 394
 395 INIT_SECTION
 396 VOID
 397 NTAPI
 398 KiInitializeKernel(IN PKPROCESS InitProcess,
 399                    IN PKTHREAD InitThread,
 400                    IN PVOID IdleStack,
 401                    IN PKPRCB Prcb,
 402                    IN CCHAR Number,
 403                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 404 {
 405     BOOLEAN NpxPresent;
 406     ULONG FeatureBits;
 407     ULONG PageDirectory[2];
 408     PVOID DpcStack;
 409     ULONG Vendor[3];
 410     KIRQL DummyIrql;
 411
 412     /* Detect and set the CPU Type */
 413     KiSetProcessorType();
 414
 415     /* Check if an FPU is present */
 416     NpxPresent = KiIsNpxPresent();
 417
 418     /* Initialize the Power Management Support for this PRCB */
 419     PoInitializePrcb(Prcb);
 420
 421     /* Bugcheck if this is a 386 CPU */
 422     if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
 423
 424     /* Get the processor features for the CPU */
 425     FeatureBits = KiGetFeatureBits();
 426
 427     /* Set the default NX policy (opt-in) */
 428     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 429
 430     /* Check if NPX is always on */
 431     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 432     {
 433         /* Set it always on */
 434         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 435         FeatureBits |= KF_NX_ENABLED;
 436     }
 437     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 438     {
 439         /* Set it in opt-out mode */
 440         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 441         FeatureBits |= KF_NX_ENABLED;
 442     }
 443     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 444              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 445     {
 446         /* Set the feature bits */
 447         FeatureBits |= KF_NX_ENABLED;
 448     }
 449     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 450              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 451     {
 452         /* Set disabled mode */
 453         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 454         FeatureBits |= KF_NX_DISABLED;
 455     }
 456
 457     /* Save feature bits */
 458     Prcb->FeatureBits = FeatureBits;
 459
 460     /* Save CPU state */
 461     KiSaveProcessorControlState(&Prcb->ProcessorState);
 462
 463     /* Get cache line information for this CPU */
 464     KiGetCacheInformation();
 465
 466     /* Initialize spinlocks and DPC data */
 467     KiInitSpinLocks(Prcb, Number);
 468
 469     /* Check if this is the Boot CPU */
 470     if (!Number)
 471     {
 472         /* Set Node Data */
 473         KeNodeBlock[0] = &KiNode0;
 474         Prcb->ParentNode = KeNodeBlock[0];
 475         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 476
 477         /* Set boot-level flags */
 478         KeI386NpxPresent = NpxPresent;
 479         KeI386CpuType = Prcb->CpuType;
 480         KeI386CpuStep = Prcb->CpuStep;
 481         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 482         KeProcessorLevel = (USHORT)Prcb->CpuType;
 483         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 484         KeFeatureBits = FeatureBits;
 485         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 486         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 487
 488         /* Detect 8-byte compare exchange support */
 489         if (!(KeFeatureBits & KF_CMPXCHG8B))
 490         {
 491             /* Copy the vendor string */
 492             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 493
 494             /* Bugcheck the system. Windows *requires* this */
 495             KeBugCheckEx(UNSUPPORTED_PROCESSOR,
 496                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 497                          Vendor[0],
 498                          Vendor[1],
 499                          Vendor[2]);
 500         }
 501
 502         /* Set the current MP Master KPRCB to the Boot PRCB */
 503         Prcb->MultiThreadSetMaster = Prcb;
 504
 505         /* Lower to APC_LEVEL */
 506         KeLowerIrql(APC_LEVEL);
 507
 508         /* Initialize some spinlocks */
 509         KeInitializeSpinLock(&KiFreezeExecutionLock);
 510         KeInitializeSpinLock(&Ki486CompatibilityLock);
 511
 512         /* Initialize portable parts of the OS */
 513         KiInitSystem();
 514
 515         /* Initialize the Idle Process and the Process Listhead */
 516         InitializeListHead(&KiProcessListHead);
 517         PageDirectory[0] = 0;
 518         PageDirectory[1] = 0;
 519         KeInitializeProcess(InitProcess,
 520                             0,
 521                             0xFFFFFFFF,
 522                             PageDirectory,
 523                             FALSE);
 524         InitProcess->QuantumReset = MAXCHAR;
 525     }
 526     else
 527     {
 528         /* FIXME */
 529         DPRINT1("SMP Boot support not yet present\n");
 530     }
 531
 532     /* Setup the Idle Thread */
 533     KeInitializeThread(InitProcess,
 534                        InitThread,
 535                        NULL,
 536                        NULL,
 537                        NULL,
 538                        NULL,
 539                        NULL,
 540                        IdleStack);
 541     InitThread->NextProcessor = Number;
 542     InitThread->Priority = HIGH_PRIORITY;
 543     InitThread->State = Running;
 544     InitThread->Affinity = 1 << Number;
 545     InitThread->WaitIrql = DISPATCH_LEVEL;
 546     InitProcess->ActiveProcessors = 1 << Number;
 547
 548     /* HACK for MmUpdatePageDir */
 549     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 550
 551     /* Set basic CPU Features that user mode can read */
 552     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 553         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 554     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 555         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 556     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 557         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 558     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 559         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 560     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 561         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 562     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 563         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 564
 565     /* Set up the thread-related fields in the PRCB */
 566     Prcb->CurrentThread = InitThread;
 567     Prcb->NextThread = NULL;
 568     Prcb->IdleThread = InitThread;
 569
 570     /* Initialize the Kernel Executive */
 571     ExpInitializeExecutive(Number, LoaderBlock);
 572
 573     /* Only do this on the boot CPU */
 574     if (!Number)
 575     {
 576         /* Calculate the time reciprocal */
 577         KiTimeIncrementReciprocal =
 578             KiComputeReciprocal(KeMaximumIncrement,
 579                                 &KiTimeIncrementShiftCount);
 580
 581         /* Update DPC Values in case they got updated by the executive */
 582         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 583         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 584         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 585
 586         /* Allocate the DPC Stack */
 587         DpcStack = MmCreateKernelStack(FALSE, 0);
 588         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 589         Prcb->DpcStack = DpcStack;
 590
 591         /* Allocate the IOPM save area. */
 592         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 593                                                   PAGE_SIZE * 2,
 594                                                   '  eK');
 595         if (!Ki386IopmSaveArea)
 596         {
 597             /* Bugcheck. We need this for V86/VDM support. */
 598             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 599         }
 600     }
 601
 602     /* Raise to Dispatch */
 603     KeRaiseIrql(DISPATCH_LEVEL,
 604                 &DummyIrql);
 605
 606     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 607     KeSetPriorityThread(InitThread, 0);
 608
 609     /* If there's no thread scheduled, put this CPU in the Idle summary */
 610     KiAcquirePrcbLock(Prcb);
 611     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 612     KiReleasePrcbLock(Prcb);
 613
 614     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 615     KeRaiseIrql(HIGH_LEVEL,
 616                 &DummyIrql);
 617     LoaderBlock->Prcb = 0;
 618 }
 619
 620 INIT_SECTION
 621 VOID
 622 FASTCALL
 623 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 624                          IN PKIDTENTRY *Idt,
 625                          IN PKIPCR *Pcr,
 626                          IN PKTSS *Tss)
 627 {
 628     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 629     KGDTENTRY TssSelector, PcrSelector;
 630     USHORT Tr, Fs;
 631
 632     /* Get GDT and IDT descriptors */
 633     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 634     __sidt(&IdtDescriptor.Limit);
 635
 636     /* Save IDT and GDT */
 637     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 638     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 639
 640     /* Get TSS and FS Selectors */
 641     Tr = Ke386GetTr();
 642     Fs = Ke386GetFs();
 643
 644     /* Get PCR Selector, mask it and get its GDT Entry */
 645     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 646
 647     /* Get the KPCR itself */
 648     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 649                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 650                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 651
 652     /* Get TSS Selector, mask it and get its GDT Entry */
 653     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 654
 655     /* Get the KTSS itself */
 656     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 657                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 658                               TssSelector.HighWord.Bytes.BaseHi << 24);
 659 }
 660
 661 INIT_SECTION
 662 VOID
 663 NTAPI
 664 KiSystemStartupBootStack(VOID)
 665 {
 666     PKTHREAD Thread;
 667
 668     /* Initialize the kernel for the current CPU */
 669     KiInitializeKernel(&KiInitialProcess.Pcb,
 670                        (PKTHREAD)KeLoaderBlock->Thread,
 671                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 672                        (PKPRCB)__readfsdword(KPCR_PRCB),
 673                        KeNumberProcessors - 1,
 674                        KeLoaderBlock);
 675
 676     /* Set the priority of this thread to 0 */
 677     Thread = KeGetCurrentThread();
 678     Thread->Priority = 0;
 679
 680     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 681     _enable();
 682     KeLowerIrql(DISPATCH_LEVEL);
 683
 684     /* Set the right wait IRQL */
 685     Thread->WaitIrql = DISPATCH_LEVEL;
 686
 687     /* Jump into the idle loop */
 688     KiIdleLoop();
 689 }
 690
 691 static
 692 VOID
 693 KiMarkPageAsReadOnly(
 694     PVOID Address)
 695 {
 696     PHARDWARE_PTE PointerPte;
 697
 698     /* Make sure the address is page aligned */
 699     ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address);
 700
 701     /* Get the PTE address */
 702     PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE);
 703     ASSERT(PointerPte->Valid);
 704     ASSERT(PointerPte->Write);
 705
 706     /* Set as read-only */
 707     PointerPte->Write = 0;
 708
 709     /* Flush the TLB entry */
 710     __invlpg(Address);
 711 }
 712
 713 INIT_SECTION
 714 VOID
 715 NTAPI
 716 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 717 {
 718     ULONG Cpu;
 719     PKTHREAD InitialThread;
 720     ULONG InitialStack;
 721     PKGDTENTRY Gdt;
 722     PKIDTENTRY Idt;
 723     KIDTENTRY NmiEntry, DoubleFaultEntry;
 724     PKTSS Tss;
 725     PKIPCR Pcr;
 726     KIRQL DummyIrql;
 727
 728     /* Boot cycles timestamp */
 729     BootCycles = __rdtsc();
 730
 731     /* Save the loader block and get the current CPU */
 732     KeLoaderBlock = LoaderBlock;
 733     Cpu = KeNumberProcessors;
 734     if (!Cpu)
 735     {
 736         /* If this is the boot CPU, set FS and the CPU Number*/
 737         Ke386SetFs(KGDT_R0_PCR);
 738         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 739
 740         /* Set the initial stack and idle thread as well */
 741         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 742         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 743     }
 744
 745     /* Save the initial thread and stack */
 746     InitialStack = LoaderBlock->KernelStack;
 747     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 748
 749     /* Clean the APC List Head */
 750     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 751
 752     /* Initialize the machine type */
 753     KiInitializeMachineType();
 754
 755     /* Skip initial setup if this isn't the Boot CPU */
 756     if (Cpu) goto AppCpuInit;
 757
 758     /* Get GDT, IDT, PCR and TSS pointers */
 759     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 760
 761     /* Setup the TSS descriptors and entries */
 762     Ki386InitializeTss(Tss, Idt, Gdt);
 763
 764     /* Initialize the PCR */
 765     RtlZeroMemory(Pcr, PAGE_SIZE);
 766     KiInitializePcr(Cpu,
 767                     Pcr,
 768                     Idt,
 769                     Gdt,
 770                     Tss,
 771                     InitialThread,
 772                     (PVOID)KiDoubleFaultStack);
 773
 774     /* Set us as the current process */
 775     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 776
 777     /* Clear DR6/7 to cleanup bootloader debugging */
 778     __writefsdword(KPCR_TEB, 0);
 779     __writefsdword(KPCR_DR6, 0);
 780     __writefsdword(KPCR_DR7, 0);
 781
 782     /* Setup the IDT */
 783     KeInitExceptions();
 784
 785     /* Load Ring 3 selectors for DS/ES */
 786     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 787     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 788
 789     /* Save NMI and double fault traps */
 790     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 791     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 792
 793     /* Copy kernel's trap handlers */
 794     RtlCopyMemory(Idt,
 795                   (PVOID)KiIdtDescriptor.Base,
 796                   KiIdtDescriptor.Limit + 1);
 797
 798     /* Restore NMI and double fault */
 799     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 800     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 801
 802 AppCpuInit:
 803     /* Loop until we can release the freeze lock */
 804     do
 805     {
 806         /* Loop until execution can continue */
 807         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 808     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 809
 810     /* Setup CPU-related fields */
 811     __writefsdword(KPCR_NUMBER, Cpu);
 812     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 813     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 814     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 815
 816     /* Initialize the Processor with HAL */
 817     HalInitializeProcessor(Cpu, KeLoaderBlock);
 818
 819     /* Set active processors */
 820     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 821     KeNumberProcessors++;
 822
 823     /* Check if this is the boot CPU */
 824     if (!Cpu)
 825     {
 826         /* Initialize debugging system */
 827         KdInitSystem(0, KeLoaderBlock);
 828
 829         /* Check for break-in */
 830         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 831
 832         /* Make the lowest page of the boot and double fault stack read-only */
 833         KiMarkPageAsReadOnly(P0BootStackData);
 834         KiMarkPageAsReadOnly(KiDoubleFaultStackData);
 835     }
 836
 837     /* Raise to HIGH_LEVEL */
 838     KeRaiseIrql(HIGH_LEVEL,
 839                 &DummyIrql);
 840
 841     /* Switch to new kernel stack and start kernel bootstrapping */
 842     KiSwitchToBootStack(InitialStack & ~3);
 843 }