ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/i386/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 INIT_SECTION
  35 VOID
  36 NTAPI
  37 KiInitMachineDependent(VOID)
  38 {
  39     ULONG CpuCount;
  40     BOOLEAN FbCaching = FALSE;
  41     NTSTATUS Status;
  42     ULONG ReturnLength;
  43     ULONG i, Affinity, Sample = 0;
  44     PFX_SAVE_AREA FxSaveArea;
  45     ULONG MXCsrMask = 0xFFBF;
  46     CPU_INFO CpuInfo;
  47     KI_SAMPLE_MAP Samples[4];
  48     PKI_SAMPLE_MAP CurrentSample = Samples;
  49     LARGE_IDENTITY_MAP IdentityMap;
  50
  51     /* Check for large page support */
  52     if (KeFeatureBits & KF_LARGE_PAGE)
  53     {
  54         /* Do an IPI to enable it on all CPUs */
  55         if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
  56             KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
  57
  58         /* Free the pages allocated for identity map */
  59         Ki386FreeIdentityMap(&IdentityMap);
  60     }
  61
  62     /* Check for global page support */
  63     if (KeFeatureBits & KF_GLOBAL_PAGE)
  64     {
  65         /* Do an IPI to enable it on all CPUs */
  66         CpuCount = KeNumberProcessors;
  67         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  68     }
  69
  70     /* Check for PAT and/or MTRR support */
  71     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  72     {
  73         /* Query the HAL to make sure we can use it */
  74         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  75                                            sizeof(BOOLEAN),
  76                                            &FbCaching,
  77                                            &ReturnLength);
  78         if ((NT_SUCCESS(Status)) && (FbCaching))
  79         {
  80             /* We can't, disable it */
  81             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  82         }
  83     }
  84
  85     /* Check for PAT support and enable it */
  86     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  87
  88     /* Assume no errata for now */
  89     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  90
  91     /* Check if we have an NPX */
  92     if (KeI386NpxPresent)
  93     {
  94         /* Loop every CPU */
  95         i = KeActiveProcessors;
  96         for (Affinity = 1; i; Affinity <<= 1)
  97         {
  98             /* Check if this is part of the set */
  99             if (i & Affinity)
 100             {
 101                 /* Run on this CPU */
 102                 i &= ~Affinity;
 103                 KeSetSystemAffinityThread(Affinity);
 104
 105                 /* Detect FPU errata */
 106                 if (KiIsNpxErrataPresent())
 107                 {
 108                     /* Disable NPX support */
 109                     KeI386NpxPresent = FALSE;
 110                     SharedUserData->
 111                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 112                         TRUE;
 113                     break;
 114                 }
 115             }
 116         }
 117     }
 118
 119     /* If there's no NPX, then we're emulating the FPU */
 120     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 121         !KeI386NpxPresent;
 122
 123     /* Check if there's no NPX, so that we can disable associated features */
 124     if (!KeI386NpxPresent)
 125     {
 126         /* Remove NPX-related bits */
 127         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 128
 129         /* Disable kernel flags */
 130         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 131
 132         /* Disable processor features that might've been set until now */
 133         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 134         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 135         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 136         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 137         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 138     }
 139
 140     /* Check for CR4 support */
 141     if (KeFeatureBits & KF_CR4)
 142     {
 143         /* Do an IPI call to enable the Debug Exceptions */
 144         CpuCount = KeNumberProcessors;
 145         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 146     }
 147
 148     /* Check if FXSR was found */
 149     if (KeFeatureBits & KF_FXSR)
 150     {
 151         /* Do an IPI call to enable the FXSR */
 152         CpuCount = KeNumberProcessors;
 153         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 154
 155         /* Check if XMM was found too */
 156         if (KeFeatureBits & KF_XMMI)
 157         {
 158             /* Do an IPI call to enable XMMI exceptions */
 159             CpuCount = KeNumberProcessors;
 160             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 161
 162             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 163
 164             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 165             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP
 166         }
 167     }
 168
 169     /* Check for, and enable SYSENTER support */
 170     KiRestoreFastSyscallReturnState();
 171
 172     /* Loop every CPU */
 173     i = KeActiveProcessors;
 174     for (Affinity = 1; i; Affinity <<= 1)
 175     {
 176         /* Check if this is part of the set */
 177         if (i & Affinity)
 178         {
 179             /* Run on this CPU */
 180             i &= ~Affinity;
 181             KeSetSystemAffinityThread(Affinity);
 182
 183             /* Reset MHz to 0 for this CPU */
 184             KeGetCurrentPrcb()->MHz = 0;
 185
 186             /* Check if we can use RDTSC */
 187             if (KeFeatureBits & KF_RDTSC)
 188             {
 189                 /* Start sampling loop */
 190                 for (;;)
 191                 {
 192                     /* Do a dummy CPUID to start the sample */
 193                     KiCpuId(&CpuInfo, 0);
 194
 195                     /* Fill out the starting data */
 196                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 197                     CurrentSample->TSCStart = __rdtsc();
 198                     CurrentSample->PerfFreq.QuadPart = -50000;
 199
 200                     /* Sleep for this sample */
 201                     KeDelayExecutionThread(KernelMode,
 202                                            FALSE,
 203                                            &CurrentSample->PerfFreq);
 204
 205                     /* Do another dummy CPUID */
 206                     KiCpuId(&CpuInfo, 0);
 207
 208                     /* Fill out the ending data */
 209                     CurrentSample->PerfEnd =
 210                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 211                     CurrentSample->TSCEnd = __rdtsc();
 212
 213                     /* Calculate the differences */
 214                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 215                                                CurrentSample->PerfStart.QuadPart;
 216                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 217                                               CurrentSample->TSCStart;
 218
 219                     /* Compute CPU Speed */
 220                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 221                                                   CurrentSample->
 222                                                   PerfFreq.QuadPart + 500000) /
 223                                                  (CurrentSample->PerfDelta *
 224                                                   1000000));
 225
 226                     /* Check if this isn't the first sample */
 227                     if (Sample)
 228                     {
 229                         /* Check if we got a good precision within 1MHz */
 230                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 231                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 232                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 233                         {
 234                             /* We did, stop sampling */
 235                             break;
 236                         }
 237                     }
 238
 239                     /* Move on */
 240                     CurrentSample++;
 241                     Sample++;
 242
 243                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 244                     {
 245                         /* Restart */
 246                         CurrentSample = Samples;
 247                         Sample = 0;
 248                     }
 249                 }
 250
 251                 /* Save the CPU Speed */
 252                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 253             }
 254
 255             /* Check if we have MTRR */
 256             if (KeFeatureBits & KF_MTRR)
 257             {
 258                 /* Then manually initialize MTRR for the CPU */
 259                 KiInitializeMTRR(i ? FALSE : TRUE);
 260             }
 261
 262             /* Check if we have AMD MTRR and initialize it for the CPU */
 263             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 264
 265             /* Check if this is a buggy Pentium and apply the fixup if so */
 266             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 267
 268             /* Check if the CPU supports FXSR */
 269             if (KeFeatureBits & KF_FXSR)
 270             {
 271                 /* Get the current thread NPX state */
 272                 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
 273
 274                 /* Clear initial MXCsr mask */
 275                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 276
 277                 /* Save the current NPX State */
 278                 Ke386SaveFpuState(FxSaveArea);
 279
 280                 /* Check if the current mask doesn't match the reserved bits */
 281                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 282                 {
 283                     /* Then use whatever it's holding */
 284                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 285                 }
 286
 287                 /* Check if nobody set the kernel-wide mask */
 288                 if (!KiMXCsrMask)
 289                 {
 290                     /* Then use the one we calculated above */
 291                     KiMXCsrMask = MXCsrMask;
 292                 }
 293                 else
 294                 {
 295                     /* Was it set to the same value we found now? */
 296                     if (KiMXCsrMask != MXCsrMask)
 297                     {
 298                         /* No, something is definitely wrong */
 299                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 300                                      KF_FXSR,
 301                                      KiMXCsrMask,
 302                                      MXCsrMask,
 303                                      0);
 304                     }
 305                 }
 306
 307                 /* Now set the kernel mask */
 308                 KiMXCsrMask &= MXCsrMask;
 309             }
 310         }
 311     }
 312
 313     /* Return affinity back to where it was */
 314     KeRevertToUserAffinityThread();
 315
 316     /* NT allows limiting the duration of an ISR with a registry key */
 317     if (KiTimeLimitIsrMicroseconds)
 318     {
 319         /* FIXME: TODO */
 320         DPRINT1("ISR Time Limit not yet supported\n");
 321     }
 322
 323     /* Set CR0 features based on detected CPU */
 324     KiSetCR0Bits();
 325 }
 326
 327 INIT_SECTION
 328 VOID
 329 NTAPI
 330 KiInitializePcr(IN ULONG ProcessorNumber,
 331                 IN PKIPCR Pcr,
 332                 IN PKIDTENTRY Idt,
 333                 IN PKGDTENTRY Gdt,
 334                 IN PKTSS Tss,
 335                 IN PKTHREAD IdleThread,
 336                 IN PVOID DpcStack)
 337 {
 338     /* Setup the TIB */
 339     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 340     Pcr->NtTib.StackBase = 0;
 341     Pcr->NtTib.StackLimit = 0;
 342     Pcr->NtTib.Self = NULL;
 343
 344     /* Set the Current Thread */
 345     Pcr->PrcbData.CurrentThread = IdleThread;
 346
 347     /* Set pointers to ourselves */
 348     Pcr->SelfPcr = (PKPCR)Pcr;
 349     Pcr->Prcb = &Pcr->PrcbData;
 350
 351     /* Set the PCR Version */
 352     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 353     Pcr->MinorVersion = PCR_MINOR_VERSION;
 354
 355     /* Set the PCRB Version */
 356     Pcr->PrcbData.MajorVersion = 1;
 357     Pcr->PrcbData.MinorVersion = 1;
 358
 359     /* Set the Build Type */
 360     Pcr->PrcbData.BuildType = 0;
 361 #ifndef CONFIG_SMP
 362     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 363 #endif
 364 #if DBG
 365     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 366 #endif
 367
 368     /* Set the Processor Number and current Processor Mask */
 369     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 370     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 371
 372     /* Set the PRCB for this Processor */
 373     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 374
 375     /* Start us out at PASSIVE_LEVEL */
 376     Pcr->Irql = PASSIVE_LEVEL;
 377
 378     /* Set the GDI, IDT, TSS and DPC Stack */
 379     Pcr->GDT = (PVOID)Gdt;
 380     Pcr->IDT = Idt;
 381     Pcr->TSS = Tss;
 382     Pcr->TssCopy = Tss;
 383     Pcr->PrcbData.DpcStack = DpcStack;
 384
 385     /* Setup the processor set */
 386     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 387 }
 388
 389 INIT_SECTION
 390 VOID
 391 NTAPI
 392 KiInitializeKernel(IN PKPROCESS InitProcess,
 393                    IN PKTHREAD InitThread,
 394                    IN PVOID IdleStack,
 395                    IN PKPRCB Prcb,
 396                    IN CCHAR Number,
 397                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 398 {
 399     BOOLEAN NpxPresent;
 400     ULONG FeatureBits;
 401     ULONG PageDirectory[2];
 402     PVOID DpcStack;
 403     ULONG Vendor[3];
 404     KIRQL DummyIrql;
 405
 406     /* Detect and set the CPU Type */
 407     KiSetProcessorType();
 408
 409     /* Check if an FPU is present */
 410     NpxPresent = KiIsNpxPresent();
 411
 412     /* Initialize the Power Management Support for this PRCB */
 413     PoInitializePrcb(Prcb);
 414
 415     /* Bugcheck if this is a 386 CPU */
 416     if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
 417
 418     /* Get the processor features for the CPU */
 419     FeatureBits = KiGetFeatureBits();
 420
 421     /* Set the default NX policy (opt-in) */
 422     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 423
 424     /* Check if NPX is always on */
 425     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 426     {
 427         /* Set it always on */
 428         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 429         FeatureBits |= KF_NX_ENABLED;
 430     }
 431     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 432     {
 433         /* Set it in opt-out mode */
 434         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 435         FeatureBits |= KF_NX_ENABLED;
 436     }
 437     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 438              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 439     {
 440         /* Set the feature bits */
 441         FeatureBits |= KF_NX_ENABLED;
 442     }
 443     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 444              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 445     {
 446         /* Set disabled mode */
 447         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 448         FeatureBits |= KF_NX_DISABLED;
 449     }
 450
 451     /* Save feature bits */
 452     Prcb->FeatureBits = FeatureBits;
 453
 454     /* Save CPU state */
 455     KiSaveProcessorControlState(&Prcb->ProcessorState);
 456
 457     /* Get cache line information for this CPU */
 458     KiGetCacheInformation();
 459
 460     /* Initialize spinlocks and DPC data */
 461     KiInitSpinLocks(Prcb, Number);
 462
 463     /* Check if this is the Boot CPU */
 464     if (!Number)
 465     {
 466         /* Set Node Data */
 467         KeNodeBlock[0] = &KiNode0;
 468         Prcb->ParentNode = KeNodeBlock[0];
 469         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 470
 471         /* Set boot-level flags */
 472         KeI386NpxPresent = NpxPresent;
 473         KeI386CpuType = Prcb->CpuType;
 474         KeI386CpuStep = Prcb->CpuStep;
 475         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 476         KeProcessorLevel = (USHORT)Prcb->CpuType;
 477         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 478         KeFeatureBits = FeatureBits;
 479         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 480         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 481
 482         /* Detect 8-byte compare exchange support */
 483         if (!(KeFeatureBits & KF_CMPXCHG8B))
 484         {
 485             /* Copy the vendor string */
 486             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 487
 488             /* Bugcheck the system. Windows *requires* this */
 489             KeBugCheckEx(UNSUPPORTED_PROCESSOR,
 490                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 491                          Vendor[0],
 492                          Vendor[1],
 493                          Vendor[2]);
 494         }
 495
 496         /* Set the current MP Master KPRCB to the Boot PRCB */
 497         Prcb->MultiThreadSetMaster = Prcb;
 498
 499         /* Lower to APC_LEVEL */
 500         KeLowerIrql(APC_LEVEL);
 501
 502         /* Initialize some spinlocks */
 503         KeInitializeSpinLock(&KiFreezeExecutionLock);
 504         KeInitializeSpinLock(&Ki486CompatibilityLock);
 505
 506         /* Initialize portable parts of the OS */
 507         KiInitSystem();
 508
 509         /* Initialize the Idle Process and the Process Listhead */
 510         InitializeListHead(&KiProcessListHead);
 511         PageDirectory[0] = 0;
 512         PageDirectory[1] = 0;
 513         KeInitializeProcess(InitProcess,
 514                             0,
 515                             0xFFFFFFFF,
 516                             PageDirectory,
 517                             FALSE);
 518         InitProcess->QuantumReset = MAXCHAR;
 519     }
 520     else
 521     {
 522         /* FIXME */
 523         DPRINT1("SMP Boot support not yet present\n");
 524     }
 525
 526     /* Setup the Idle Thread */
 527     KeInitializeThread(InitProcess,
 528                        InitThread,
 529                        NULL,
 530                        NULL,
 531                        NULL,
 532                        NULL,
 533                        NULL,
 534                        IdleStack);
 535     InitThread->NextProcessor = Number;
 536     InitThread->Priority = HIGH_PRIORITY;
 537     InitThread->State = Running;
 538     InitThread->Affinity = 1 << Number;
 539     InitThread->WaitIrql = DISPATCH_LEVEL;
 540     InitProcess->ActiveProcessors = 1 << Number;
 541
 542     /* HACK for MmUpdatePageDir */
 543     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 544
 545     /* Set basic CPU Features that user mode can read */
 546     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 547         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 548     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 549         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 550     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 551         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 552     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 553         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 554     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 555         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 556     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 557         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 558
 559     /* Set up the thread-related fields in the PRCB */
 560     Prcb->CurrentThread = InitThread;
 561     Prcb->NextThread = NULL;
 562     Prcb->IdleThread = InitThread;
 563
 564     /* Initialize the Kernel Executive */
 565     ExpInitializeExecutive(Number, LoaderBlock);
 566
 567     /* Only do this on the boot CPU */
 568     if (!Number)
 569     {
 570         /* Calculate the time reciprocal */
 571         KiTimeIncrementReciprocal =
 572             KiComputeReciprocal(KeMaximumIncrement,
 573                                 &KiTimeIncrementShiftCount);
 574
 575         /* Update DPC Values in case they got updated by the executive */
 576         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 577         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 578         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 579
 580         /* Allocate the DPC Stack */
 581         DpcStack = MmCreateKernelStack(FALSE, 0);
 582         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 583         Prcb->DpcStack = DpcStack;
 584
 585         /* Allocate the IOPM save area. */
 586         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 587                                                   PAGE_SIZE * 2,
 588                                                   '  eK');
 589         if (!Ki386IopmSaveArea)
 590         {
 591             /* Bugcheck. We need this for V86/VDM support. */
 592             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 593         }
 594     }
 595
 596     /* Raise to Dispatch */
 597     KeRaiseIrql(DISPATCH_LEVEL,
 598                 &DummyIrql);
 599
 600     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 601     KeSetPriorityThread(InitThread, 0);
 602
 603     /* If there's no thread scheduled, put this CPU in the Idle summary */
 604     KiAcquirePrcbLock(Prcb);
 605     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 606     KiReleasePrcbLock(Prcb);
 607
 608     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 609     KeRaiseIrql(HIGH_LEVEL,
 610                 &DummyIrql);
 611     LoaderBlock->Prcb = 0;
 612 }
 613
 614 INIT_SECTION
 615 VOID
 616 FASTCALL
 617 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 618                          IN PKIDTENTRY *Idt,
 619                          IN PKIPCR *Pcr,
 620                          IN PKTSS *Tss)
 621 {
 622     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 623     KGDTENTRY TssSelector, PcrSelector;
 624     USHORT Tr, Fs;
 625
 626     /* Get GDT and IDT descriptors */
 627     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 628     __sidt(&IdtDescriptor.Limit);
 629
 630     /* Save IDT and GDT */
 631     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 632     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 633
 634     /* Get TSS and FS Selectors */
 635     Tr = Ke386GetTr();
 636     Fs = Ke386GetFs();
 637
 638     /* Get PCR Selector, mask it and get its GDT Entry */
 639     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 640
 641     /* Get the KPCR itself */
 642     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 643                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 644                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 645
 646     /* Get TSS Selector, mask it and get its GDT Entry */
 647     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 648
 649     /* Get the KTSS itself */
 650     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 651                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 652                               TssSelector.HighWord.Bytes.BaseHi << 24);
 653 }
 654
 655 INIT_SECTION
 656 VOID
 657 NTAPI
 658 KiSystemStartupBootStack(VOID)
 659 {
 660     PKTHREAD Thread;
 661
 662     /* Initialize the kernel for the current CPU */
 663     KiInitializeKernel(&KiInitialProcess.Pcb,
 664                        (PKTHREAD)KeLoaderBlock->Thread,
 665                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 666                        (PKPRCB)__readfsdword(KPCR_PRCB),
 667                        KeNumberProcessors - 1,
 668                        KeLoaderBlock);
 669
 670     /* Set the priority of this thread to 0 */
 671     Thread = KeGetCurrentThread();
 672     Thread->Priority = 0;
 673
 674     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 675     _enable();
 676     KeLowerIrql(DISPATCH_LEVEL);
 677
 678     /* Set the right wait IRQL */
 679     Thread->WaitIrql = DISPATCH_LEVEL;
 680
 681     /* Jump into the idle loop */
 682     KiIdleLoop();
 683 }
 684
 685 static
 686 VOID
 687 KiMarkPageAsReadOnly(
 688     PVOID Address)
 689 {
 690     PHARDWARE_PTE PointerPte;
 691
 692     /* Make sure the address is page aligned */
 693     ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address);
 694
 695     /* Get the PTE address */
 696     PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE);
 697     ASSERT(PointerPte->Valid);
 698     ASSERT(PointerPte->Write);
 699
 700     /* Set as read-only */
 701     PointerPte->Write = 0;
 702
 703     /* Flush the TLB entry */
 704     __invlpg(Address);
 705 }
 706
 707 INIT_SECTION
 708 VOID
 709 NTAPI
 710 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 711 {
 712     ULONG Cpu;
 713     PKTHREAD InitialThread;
 714     ULONG InitialStack;
 715     PKGDTENTRY Gdt;
 716     PKIDTENTRY Idt;
 717     KIDTENTRY NmiEntry, DoubleFaultEntry;
 718     PKTSS Tss;
 719     PKIPCR Pcr;
 720     KIRQL DummyIrql;
 721
 722     /* Boot cycles timestamp */
 723     BootCycles = __rdtsc();
 724
 725     /* Save the loader block and get the current CPU */
 726     KeLoaderBlock = LoaderBlock;
 727     Cpu = KeNumberProcessors;
 728     if (!Cpu)
 729     {
 730         /* If this is the boot CPU, set FS and the CPU Number*/
 731         Ke386SetFs(KGDT_R0_PCR);
 732         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 733
 734         /* Set the initial stack and idle thread as well */
 735         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 736         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 737     }
 738
 739     /* Save the initial thread and stack */
 740     InitialStack = LoaderBlock->KernelStack;
 741     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 742
 743     /* Clean the APC List Head */
 744     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 745
 746     /* Initialize the machine type */
 747     KiInitializeMachineType();
 748
 749     /* Skip initial setup if this isn't the Boot CPU */
 750     if (Cpu) goto AppCpuInit;
 751
 752     /* Get GDT, IDT, PCR and TSS pointers */
 753     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 754
 755     /* Setup the TSS descriptors and entries */
 756     Ki386InitializeTss(Tss, Idt, Gdt);
 757
 758     /* Initialize the PCR */
 759     RtlZeroMemory(Pcr, PAGE_SIZE);
 760     KiInitializePcr(Cpu,
 761                     Pcr,
 762                     Idt,
 763                     Gdt,
 764                     Tss,
 765                     InitialThread,
 766                     (PVOID)KiDoubleFaultStack);
 767
 768     /* Set us as the current process */
 769     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 770
 771     /* Clear DR6/7 to cleanup bootloader debugging */
 772     __writefsdword(KPCR_TEB, 0);
 773     __writefsdword(KPCR_DR6, 0);
 774     __writefsdword(KPCR_DR7, 0);
 775
 776     /* Setup the IDT */
 777     KeInitExceptions();
 778
 779     /* Load Ring 3 selectors for DS/ES */
 780     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 781     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 782
 783     /* Save NMI and double fault traps */
 784     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 785     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 786
 787     /* Copy kernel's trap handlers */
 788     RtlCopyMemory(Idt,
 789                   (PVOID)KiIdtDescriptor.Base,
 790                   KiIdtDescriptor.Limit + 1);
 791
 792     /* Restore NMI and double fault */
 793     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 794     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 795
 796 AppCpuInit:
 797     /* Loop until we can release the freeze lock */
 798     do
 799     {
 800         /* Loop until execution can continue */
 801         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 802     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 803
 804     /* Setup CPU-related fields */
 805     __writefsdword(KPCR_NUMBER, Cpu);
 806     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 807     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 808     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 809
 810     /* Initialize the Processor with HAL */
 811     HalInitializeProcessor(Cpu, KeLoaderBlock);
 812
 813     /* Set active processors */
 814     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 815     KeNumberProcessors++;
 816
 817     /* Check if this is the boot CPU */
 818     if (!Cpu)
 819     {
 820         /* Initialize debugging system */
 821         KdInitSystem(0, KeLoaderBlock);
 822
 823         /* Check for break-in */
 824         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 825
 826         /* Make the lowest page of the boot and double fault stack read-only */
 827         KiMarkPageAsReadOnly(P0BootStackData);
 828         KiMarkPageAsReadOnly(KiDoubleFaultStackData);
 829     }
 830
 831     /* Raise to HIGH_LEVEL */
 832     KeRaiseIrql(HIGH_LEVEL,
 833                 &DummyIrql);
 834
 835     /* Switch to new kernel stack and start kernel bootstrapping */
 836     KiSwitchToBootStack(InitialStack & ~3);
 837 }