ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/i386/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  20 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 VOID
  35 NTAPI
  36 INIT_FUNCTION
  37 KiInitMachineDependent(VOID)
  38 {
  39     ULONG CpuCount;
  40     BOOLEAN FbCaching = FALSE;
  41     NTSTATUS Status;
  42     ULONG ReturnLength;
  43     ULONG i, Affinity, Sample = 0;
  44     PFX_SAVE_AREA FxSaveArea;
  45     ULONG MXCsrMask = 0xFFBF;
  46     ULONG Dummy;
  47     KI_SAMPLE_MAP Samples[4];
  48     PKI_SAMPLE_MAP CurrentSample = Samples;
  49     LARGE_IDENTITY_MAP IdentityMap;
  50
  51     /* Check for large page support */
  52     if (KeFeatureBits & KF_LARGE_PAGE)
  53     {
  54         /* Do an IPI to enable it on all CPUs */
  55         if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
  56             KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
  57
  58         /* Free the pages allocated for identity map */
  59         Ki386FreeIdentityMap(&IdentityMap);
  60     }
  61
  62     /* Check for global page support */
  63     if (KeFeatureBits & KF_GLOBAL_PAGE)
  64     {
  65         /* Do an IPI to enable it on all CPUs */
  66         CpuCount = KeNumberProcessors;
  67         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  68     }
  69
  70     /* Check for PAT and/or MTRR support */
  71     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  72     {
  73         /* Query the HAL to make sure we can use it */
  74         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  75                                            sizeof(BOOLEAN),
  76                                            &FbCaching,
  77                                            &ReturnLength);
  78         if ((NT_SUCCESS(Status)) && (FbCaching))
  79         {
  80             /* We can't, disable it */
  81             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  82         }
  83     }
  84
  85     /* Check for PAT support and enable it */
  86     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  87
  88     /* Assume no errata for now */
  89     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  90
  91     /* Check if we have an NPX */
  92     if (KeI386NpxPresent)
  93     {
  94         /* Loop every CPU */
  95         i = KeActiveProcessors;
  96         for (Affinity = 1; i; Affinity <<= 1)
  97         {
  98             /* Check if this is part of the set */
  99             if (i & Affinity)
 100             {
 101                 /* Run on this CPU */
 102                 i &= ~Affinity;
 103                 KeSetSystemAffinityThread(Affinity);
 104
 105                 /* Detect FPU errata */
 106                 if (KiIsNpxErrataPresent())
 107                 {
 108                     /* Disable NPX support */
 109                     KeI386NpxPresent = FALSE;
 110                     SharedUserData->
 111                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 112                         TRUE;
 113                     break;
 114                 }
 115             }
 116         }
 117     }
 118
 119     /* If there's no NPX, then we're emulating the FPU */
 120     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 121         !KeI386NpxPresent;
 122
 123     /* Check if there's no NPX, so that we can disable associated features */
 124     if (!KeI386NpxPresent)
 125     {
 126         /* Remove NPX-related bits */
 127         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 128
 129         /* Disable kernel flags */
 130         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 131
 132         /* Disable processor features that might've been set until now */
 133         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 134         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 135         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 136         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 137         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 138     }
 139
 140     /* Check for CR4 support */
 141     if (KeFeatureBits & KF_CR4)
 142     {
 143         /* Do an IPI call to enable the Debug Exceptions */
 144         CpuCount = KeNumberProcessors;
 145         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 146     }
 147
 148     /* Check if FXSR was found */
 149     if (KeFeatureBits & KF_FXSR)
 150     {
 151         /* Do an IPI call to enable the FXSR */
 152         CpuCount = KeNumberProcessors;
 153         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 154
 155         /* Check if XMM was found too */
 156         if (KeFeatureBits & KF_XMMI)
 157         {
 158             /* Do an IPI call to enable XMMI exceptions */
 159             CpuCount = KeNumberProcessors;
 160             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 161
 162             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 163
 164             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 165             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 166         }
 167     }
 168
 169     /* Check for, and enable SYSENTER support */
 170     KiRestoreFastSyscallReturnState();
 171
 172     /* Loop every CPU */
 173     i = KeActiveProcessors;
 174     for (Affinity = 1; i; Affinity <<= 1)
 175     {
 176         /* Check if this is part of the set */
 177         if (i & Affinity)
 178         {
 179             /* Run on this CPU */
 180             i &= ~Affinity;
 181             KeSetSystemAffinityThread(Affinity);
 182
 183             /* Reset MHz to 0 for this CPU */
 184             KeGetCurrentPrcb()->MHz = 0;
 185
 186             /* Check if we can use RDTSC */
 187             if (KeFeatureBits & KF_RDTSC)
 188             {
 189                 /* Start sampling loop */
 190                 for (;;)
 191                 {
 192                     /* Do a dummy CPUID to start the sample */
 193                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 194
 195                     /* Fill out the starting data */
 196                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 197                     CurrentSample->TSCStart = __rdtsc();
 198                     CurrentSample->PerfFreq.QuadPart = -50000;
 199
 200                     /* Sleep for this sample */
 201                     KeDelayExecutionThread(KernelMode,
 202                                            FALSE,
 203                                            &CurrentSample->PerfFreq);
 204
 205                     /* Do another dummy CPUID */
 206                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 207
 208                     /* Fill out the ending data */
 209                     CurrentSample->PerfEnd =
 210                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 211                     CurrentSample->TSCEnd = __rdtsc();
 212
 213                     /* Calculate the differences */
 214                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 215                                                CurrentSample->PerfStart.QuadPart;
 216                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 217                                               CurrentSample->TSCStart;
 218
 219                     /* Compute CPU Speed */
 220                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 221                                                   CurrentSample->
 222                                                   PerfFreq.QuadPart + 500000) /
 223                                                  (CurrentSample->PerfDelta *
 224                                                   1000000));
 225
 226                     /* Check if this isn't the first sample */
 227                     if (Sample)
 228                     {
 229                         /* Check if we got a good precision within 1MHz */
 230                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 231                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 232                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 233                         {
 234                             /* We did, stop sampling */
 235                             break;
 236                         }
 237                     }
 238
 239                     /* Move on */
 240                     CurrentSample++;
 241                     Sample++;
 242
 243                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 244                     {
 245                         /* Restart */
 246                         CurrentSample = Samples;
 247                         Sample = 0;
 248                     }
 249                 }
 250
 251                 /* Save the CPU Speed */
 252                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 253             }
 254
 255             /* Check if we have MTRR */
 256             if (KeFeatureBits & KF_MTRR)
 257             {
 258                 /* Then manually initialize MTRR for the CPU */
 259                 KiInitializeMTRR(i ? FALSE : TRUE);
 260             }
 261
 262             /* Check if we have AMD MTRR and initialize it for the CPU */
 263             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 264
 265             /* Check if this is a buggy Pentium and apply the fixup if so */
 266             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 267
 268             /* Check if the CPU supports FXSR */
 269             if (KeFeatureBits & KF_FXSR)
 270             {
 271                 /* Get the current thread NPX state */
 272                 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
 273
 274                 /* Clear initial MXCsr mask */
 275                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 276
 277                 /* Save the current NPX State */
 278                 Ke386SaveFpuState(FxSaveArea);
 279
 280                 /* Check if the current mask doesn't match the reserved bits */
 281                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 282                 {
 283                     /* Then use whatever it's holding */
 284                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 285                 }
 286
 287                 /* Check if nobody set the kernel-wide mask */
 288                 if (!KiMXCsrMask)
 289                 {
 290                     /* Then use the one we calculated above */
 291                     KiMXCsrMask = MXCsrMask;
 292                 }
 293                 else
 294                 {
 295                     /* Was it set to the same value we found now? */
 296                     if (KiMXCsrMask != MXCsrMask)
 297                     {
 298                         /* No, something is definitely wrong */
 299                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 300                                      KF_FXSR,
 301                                      KiMXCsrMask,
 302                                      MXCsrMask,
 303                                      0);
 304                     }
 305                 }
 306
 307                 /* Now set the kernel mask */
 308                 KiMXCsrMask &= MXCsrMask;
 309             }
 310         }
 311     }
 312
 313     /* Return affinity back to where it was */
 314     KeRevertToUserAffinityThread();
 315
 316     /* NT allows limiting the duration of an ISR with a registry key */
 317     if (KiTimeLimitIsrMicroseconds)
 318     {
 319         /* FIXME: TODO */
 320         DPRINT1("ISR Time Limit not yet supported\n");
 321     }
 322
 323     /* Set CR0 features based on detected CPU */
 324     KiSetCR0Bits();
 325 }
 326
 327 VOID
 328 NTAPI
 329 INIT_FUNCTION
 330 KiInitializePcr(IN ULONG ProcessorNumber,
 331                 IN PKIPCR Pcr,
 332                 IN PKIDTENTRY Idt,
 333                 IN PKGDTENTRY Gdt,
 334                 IN PKTSS Tss,
 335                 IN PKTHREAD IdleThread,
 336                 IN PVOID DpcStack)
 337 {
 338     /* Setup the TIB */
 339     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 340     Pcr->NtTib.StackBase = 0;
 341     Pcr->NtTib.StackLimit = 0;
 342     Pcr->NtTib.Self = NULL;
 343
 344     /* Set the Current Thread */
 345     Pcr->PrcbData.CurrentThread = IdleThread;
 346
 347     /* Set pointers to ourselves */
 348     Pcr->Self = (PKPCR)Pcr;
 349     Pcr->Prcb = &Pcr->PrcbData;
 350
 351     /* Set the PCR Version */
 352     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 353     Pcr->MinorVersion = PCR_MINOR_VERSION;
 354
 355     /* Set the PCRB Version */
 356     Pcr->PrcbData.MajorVersion = 1;
 357     Pcr->PrcbData.MinorVersion = 1;
 358
 359     /* Set the Build Type */
 360     Pcr->PrcbData.BuildType = 0;
 361 #ifndef CONFIG_SMP
 362     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 363 #endif
 364 #if DBG
 365     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 366 #endif
 367
 368     /* Set the Processor Number and current Processor Mask */
 369     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 370     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 371
 372     /* Set the PRCB for this Processor */
 373     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 374
 375     /* Start us out at PASSIVE_LEVEL */
 376     Pcr->Irql = PASSIVE_LEVEL;
 377
 378     /* Set the GDI, IDT, TSS and DPC Stack */
 379     Pcr->GDT = (PVOID)Gdt;
 380     Pcr->IDT = Idt;
 381     Pcr->TSS = Tss;
 382     Pcr->TssCopy = Tss;
 383     Pcr->PrcbData.DpcStack = DpcStack;
 384
 385     /* Setup the processor set */
 386     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 387 }
 388
 389 VOID
 390 NTAPI
 391 INIT_FUNCTION
 392 KiInitializeKernel(IN PKPROCESS InitProcess,
 393                    IN PKTHREAD InitThread,
 394                    IN PVOID IdleStack,
 395                    IN PKPRCB Prcb,
 396                    IN CCHAR Number,
 397                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 398 {
 399     BOOLEAN NpxPresent;
 400     ULONG FeatureBits;
 401     ULONG PageDirectory[2];
 402     PVOID DpcStack;
 403     ULONG Vendor[3];
 404
 405     /* Detect and set the CPU Type */
 406     KiSetProcessorType();
 407
 408     /* Check if an FPU is present */
 409     NpxPresent = KiIsNpxPresent();
 410
 411     /* Initialize the Power Management Support for this PRCB */
 412     PoInitializePrcb(Prcb);
 413
 414     /* Bugcheck if this is a 386 CPU */
 415     if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
 416
 417     /* Get the processor features for the CPU */
 418     FeatureBits = KiGetFeatureBits();
 419
 420     /* Set the default NX policy (opt-in) */
 421     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 422
 423     /* Check if NPX is always on */
 424     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 425     {
 426         /* Set it always on */
 427         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 428         FeatureBits |= KF_NX_ENABLED;
 429     }
 430     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 431     {
 432         /* Set it in opt-out mode */
 433         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 434         FeatureBits |= KF_NX_ENABLED;
 435     }
 436     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 437              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 438     {
 439         /* Set the feature bits */
 440         FeatureBits |= KF_NX_ENABLED;
 441     }
 442     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 443              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 444     {
 445         /* Set disabled mode */
 446         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 447         FeatureBits |= KF_NX_DISABLED;
 448     }
 449
 450     /* Save feature bits */
 451     Prcb->FeatureBits = FeatureBits;
 452
 453     /* Save CPU state */
 454     KiSaveProcessorControlState(&Prcb->ProcessorState);
 455
 456     /* Get cache line information for this CPU */
 457     KiGetCacheInformation();
 458
 459     /* Initialize spinlocks and DPC data */
 460     KiInitSpinLocks(Prcb, Number);
 461
 462     /* Check if this is the Boot CPU */
 463     if (!Number)
 464     {
 465         /* Set Node Data */
 466         KeNodeBlock[0] = &KiNode0;
 467         Prcb->ParentNode = KeNodeBlock[0];
 468         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 469
 470         /* Set boot-level flags */
 471         KeI386NpxPresent = NpxPresent;
 472         KeI386CpuType = Prcb->CpuType;
 473         KeI386CpuStep = Prcb->CpuStep;
 474         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 475         KeProcessorLevel = (USHORT)Prcb->CpuType;
 476         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 477         KeFeatureBits = FeatureBits;
 478         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 479         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 480
 481         /* Detect 8-byte compare exchange support */
 482         if (!(KeFeatureBits & KF_CMPXCHG8B))
 483         {
 484             /* Copy the vendor string */
 485             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 486
 487             /* Bugcheck the system. Windows *requires* this */
 488             KeBugCheckEx(UNSUPPORTED_PROCESSOR,
 489                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 490                          Vendor[0],
 491                          Vendor[1],
 492                          Vendor[2]);
 493         }
 494
 495         /* Set the current MP Master KPRCB to the Boot PRCB */
 496         Prcb->MultiThreadSetMaster = Prcb;
 497
 498         /* Lower to APC_LEVEL */
 499         KeLowerIrql(APC_LEVEL);
 500
 501         /* Initialize some spinlocks */
 502         KeInitializeSpinLock(&KiFreezeExecutionLock);
 503         KeInitializeSpinLock(&Ki486CompatibilityLock);
 504
 505         /* Initialize portable parts of the OS */
 506         KiInitSystem();
 507
 508         /* Initialize the Idle Process and the Process Listhead */
 509         InitializeListHead(&KiProcessListHead);
 510         PageDirectory[0] = 0;
 511         PageDirectory[1] = 0;
 512         KeInitializeProcess(InitProcess,
 513                             0,
 514                             0xFFFFFFFF,
 515                             PageDirectory,
 516                             FALSE);
 517         InitProcess->QuantumReset = MAXCHAR;
 518     }
 519     else
 520     {
 521         /* FIXME */
 522         DPRINT1("SMP Boot support not yet present\n");
 523     }
 524
 525     /* Setup the Idle Thread */
 526     KeInitializeThread(InitProcess,
 527                        InitThread,
 528                        NULL,
 529                        NULL,
 530                        NULL,
 531                        NULL,
 532                        NULL,
 533                        IdleStack);
 534     InitThread->NextProcessor = Number;
 535     InitThread->Priority = HIGH_PRIORITY;
 536     InitThread->State = Running;
 537     InitThread->Affinity = 1 << Number;
 538     InitThread->WaitIrql = DISPATCH_LEVEL;
 539     InitProcess->ActiveProcessors = 1 << Number;
 540
 541     /* HACK for MmUpdatePageDir */
 542     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 543
 544     /* Set basic CPU Features that user mode can read */
 545     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 546         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 547     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 548         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 549     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 550         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 551     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 552         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 553     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 554         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 555     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 556         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 557
 558     /* Set up the thread-related fields in the PRCB */
 559     Prcb->CurrentThread = InitThread;
 560     Prcb->NextThread = NULL;
 561     Prcb->IdleThread = InitThread;
 562
 563     /* Initialize the Kernel Executive */
 564     ExpInitializeExecutive(Number, LoaderBlock);
 565
 566     /* Only do this on the boot CPU */
 567     if (!Number)
 568     {
 569         /* Calculate the time reciprocal */
 570         KiTimeIncrementReciprocal =
 571             KiComputeReciprocal(KeMaximumIncrement,
 572                                 &KiTimeIncrementShiftCount);
 573
 574         /* Update DPC Values in case they got updated by the executive */
 575         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 576         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 577         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 578
 579         /* Allocate the DPC Stack */
 580         DpcStack = MmCreateKernelStack(FALSE, 0);
 581         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 582         Prcb->DpcStack = DpcStack;
 583
 584         /* Allocate the IOPM save area. */
 585         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 586                                                   PAGE_SIZE * 2,
 587                                                   '  eK');
 588         if (!Ki386IopmSaveArea)
 589         {
 590             /* Bugcheck. We need this for V86/VDM support. */
 591             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 592         }
 593     }
 594
 595     /* Raise to Dispatch */
 596     KfRaiseIrql(DISPATCH_LEVEL);
 597
 598     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 599     KeSetPriorityThread(InitThread, 0);
 600
 601     /* If there's no thread scheduled, put this CPU in the Idle summary */
 602     KiAcquirePrcbLock(Prcb);
 603     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 604     KiReleasePrcbLock(Prcb);
 605
 606     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 607     KfRaiseIrql(HIGH_LEVEL);
 608     LoaderBlock->Prcb = 0;
 609 }
 610
 611 VOID
 612 FASTCALL
 613 INIT_FUNCTION
 614 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 615                          IN PKIDTENTRY *Idt,
 616                          IN PKIPCR *Pcr,
 617                          IN PKTSS *Tss)
 618 {
 619     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 620     KGDTENTRY TssSelector, PcrSelector;
 621     USHORT Tr, Fs;
 622
 623     /* Get GDT and IDT descriptors */
 624     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 625     __sidt(&IdtDescriptor.Limit);
 626
 627     /* Save IDT and GDT */
 628     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 629     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 630
 631     /* Get TSS and FS Selectors */
 632     Tr = Ke386GetTr();
 633     Fs = Ke386GetFs();
 634
 635     /* Get PCR Selector, mask it and get its GDT Entry */
 636     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 637
 638     /* Get the KPCR itself */
 639     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 640                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 641                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 642
 643     /* Get TSS Selector, mask it and get its GDT Entry */
 644     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 645
 646     /* Get the KTSS itself */
 647     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 648                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 649                               TssSelector.HighWord.Bytes.BaseHi << 24);
 650 }
 651
 652 VOID
 653 NTAPI
 654 INIT_FUNCTION
 655 KiSystemStartupBootStack(VOID)
 656 {
 657     PKTHREAD Thread;
 658
 659     /* Initialize the kernel for the current CPU */
 660     KiInitializeKernel(&KiInitialProcess.Pcb,
 661                        (PKTHREAD)KeLoaderBlock->Thread,
 662                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 663                        (PKPRCB)__readfsdword(KPCR_PRCB),
 664                        KeNumberProcessors - 1,
 665                        KeLoaderBlock);
 666
 667     /* Set the priority of this thread to 0 */
 668     Thread = KeGetCurrentThread();
 669     Thread->Priority = 0;
 670
 671     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 672     _enable();
 673     KfLowerIrql(DISPATCH_LEVEL);
 674
 675     /* Set the right wait IRQL */
 676     Thread->WaitIrql = DISPATCH_LEVEL;
 677
 678     /* Jump into the idle loop */
 679     KiIdleLoop();
 680 }
 681
 682 VOID
 683 NTAPI
 684 INIT_FUNCTION
 685 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 686 {
 687     ULONG Cpu;
 688     PKTHREAD InitialThread;
 689     ULONG InitialStack;
 690     PKGDTENTRY Gdt;
 691     PKIDTENTRY Idt;
 692     KIDTENTRY NmiEntry, DoubleFaultEntry;
 693     PKTSS Tss;
 694     PKIPCR Pcr;
 695
 696     /* Boot cycles timestamp */
 697     BootCycles = __rdtsc();
 698
 699     /* Save the loader block and get the current CPU */
 700     KeLoaderBlock = LoaderBlock;
 701     Cpu = KeNumberProcessors;
 702     if (!Cpu)
 703     {
 704         /* If this is the boot CPU, set FS and the CPU Number*/
 705         Ke386SetFs(KGDT_R0_PCR);
 706         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 707
 708         /* Set the initial stack and idle thread as well */
 709         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 710         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 711     }
 712
 713     /* Save the initial thread and stack */
 714     InitialStack = LoaderBlock->KernelStack;
 715     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 716
 717     /* Clean the APC List Head */
 718     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 719
 720     /* Initialize the machine type */
 721     KiInitializeMachineType();
 722
 723     /* Skip initial setup if this isn't the Boot CPU */
 724     if (Cpu) goto AppCpuInit;
 725
 726     /* Get GDT, IDT, PCR and TSS pointers */
 727     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 728
 729     /* Setup the TSS descriptors and entries */
 730     Ki386InitializeTss(Tss, Idt, Gdt);
 731
 732     /* Initialize the PCR */
 733     RtlZeroMemory(Pcr, PAGE_SIZE);
 734     KiInitializePcr(Cpu,
 735                     Pcr,
 736                     Idt,
 737                     Gdt,
 738                     Tss,
 739                     InitialThread,
 740                     (PVOID)KiDoubleFaultStack);
 741
 742     /* Set us as the current process */
 743     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 744
 745     /* Clear DR6/7 to cleanup bootloader debugging */
 746     __writefsdword(KPCR_TEB, 0);
 747     __writefsdword(KPCR_DR6, 0);
 748     __writefsdword(KPCR_DR7, 0);
 749
 750     /* Setup the IDT */
 751     KeInitExceptions();
 752
 753     /* Load Ring 3 selectors for DS/ES */
 754     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 755     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 756
 757     /* Save NMI and double fault traps */
 758     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 759     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 760
 761     /* Copy kernel's trap handlers */
 762     RtlCopyMemory(Idt,
 763                   (PVOID)KiIdtDescriptor.Base,
 764                   KiIdtDescriptor.Limit + 1);
 765
 766     /* Restore NMI and double fault */
 767     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 768     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 769
 770 AppCpuInit:
 771     /* Loop until we can release the freeze lock */
 772     do
 773     {
 774         /* Loop until execution can continue */
 775         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 776     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 777
 778     /* Setup CPU-related fields */
 779     __writefsdword(KPCR_NUMBER, Cpu);
 780     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 781     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 782     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 783
 784     /* Initialize the Processor with HAL */
 785     HalInitializeProcessor(Cpu, KeLoaderBlock);
 786
 787     /* Set active processors */
 788     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 789     KeNumberProcessors++;
 790
 791     /* Check if this is the boot CPU */
 792     if (!Cpu)
 793     {
 794         /* Initialize debugging system */
 795         KdInitSystem(0, KeLoaderBlock);
 796
 797         /* Check for break-in */
 798         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 799     }
 800
 801     /* Raise to HIGH_LEVEL */
 802     KfRaiseIrql(HIGH_LEVEL);
 803
 804     /* Switch to new kernel stack and start kernel bootstrapping */
 805     KiSwitchToBootStack(InitialStack & ~3);
 806 }