reactos/ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* Spinlocks used only on X86 */
  18 KSPIN_LOCK KiFreezeExecutionLock;
  19 KSPIN_LOCK Ki486CompatibilityLock;
  20
  21 /* FUNCTIONS *****************************************************************/
  22
  23 VOID
  24 NTAPI
  25 KiInitMachineDependent(VOID)
  26 {
  27     ULONG Protect;
  28     ULONG CpuCount;
  29     BOOLEAN FbCaching = FALSE;
  30     NTSTATUS Status;
  31     ULONG ReturnLength;
  32     ULONG i, Affinity, Sample = 0;
  33     PFX_SAVE_AREA FxSaveArea;
  34     ULONG MXCsrMask = 0xFFBF;
  35     ULONG Dummy[4];
  36     KI_SAMPLE_MAP Samples[4];
  37     PKI_SAMPLE_MAP CurrentSample = Samples;
  38
  39     /* Check for large page support */
  40     if (KeFeatureBits & KF_LARGE_PAGE)
  41     {
  42         /* FIXME: Support this */
  43         DPRINT1("Large Page support detected but not yet taken advantage of!\n");
  44     }
  45
  46     /* Check for global page support */
  47     if (KeFeatureBits & KF_GLOBAL_PAGE)
  48     {
  49         /* Do an IPI to enable it on all CPUs */
  50         CpuCount = KeNumberProcessors;
  51         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  52     }
  53
  54     /* Check for PAT and/or MTRR support */
  55     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  56     {
  57         /* Query the HAL to make sure we can use it */
  58         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  59                                            sizeof(BOOLEAN),
  60                                            &FbCaching,
  61                                            &ReturnLength);
  62         if ((NT_SUCCESS(Status)) && (FbCaching))
  63         {
  64             /* We can't, disable it */
  65             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  66         }
  67     }
  68
  69     /* Check for PAT support and enable it */
  70     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  71
  72     /* Assume no errata for now */
  73     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  74
  75     /* Check if we have an NPX */
  76     if (KeI386NpxPresent)
  77     {
  78         /* Loop every CPU */
  79         i = KeActiveProcessors;
  80         for (Affinity = 1; i; Affinity <<= 1)
  81         {
  82             /* Check if this is part of the set */
  83             if (i & Affinity)
  84             {
  85                 /* Run on this CPU */
  86                 i &= ~Affinity;
  87                 KeSetSystemAffinityThread(Affinity);
  88
  89                 /* Detect FPU errata */
  90                 if (KiIsNpxErrataPresent())
  91                 {
  92                     /* Disable NPX support */
  93                     KeI386NpxPresent = FALSE;
  94                     SharedUserData->
  95                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
  96                         TRUE;
  97                     break;
  98                 }
  99             }
 100         }
 101     }
 102
 103     /* If there's no NPX, then we're emulating the FPU */
 104     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 105         !KeI386NpxPresent;
 106
 107     /* Check if there's no NPX, so that we can disable associated features */
 108     if (!KeI386NpxPresent)
 109     {
 110         /* Remove NPX-related bits */
 111         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 112
 113         /* Disable kernel flags */
 114         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 115
 116         /* Disable processor features that might've been set until now */
 117         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 118         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 119         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 120         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 121         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 122     }
 123
 124     /* Check for CR4 support */
 125     if (KeFeatureBits & KF_CR4)
 126     {
 127         /* Do an IPI call to enable the Debug Exceptions */
 128         CpuCount = KeNumberProcessors;
 129         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 130     }
 131
 132     /* Check if FXSR was found */
 133     if (KeFeatureBits & KF_FXSR)
 134     {
 135         /* Do an IPI call to enable the FXSR */
 136         CpuCount = KeNumberProcessors;
 137         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 138
 139         /* Check if XMM was found too */
 140         if (KeFeatureBits & KF_XMMI)
 141         {
 142             /* Do an IPI call to enable XMMI exceptions */
 143             CpuCount = KeNumberProcessors;
 144             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 145
 146             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 147
 148             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 149             Protect = MmGetPageProtect(NULL, RtlPrefetchMemoryNonTemporal);
 150             MmSetPageProtect(NULL,
 151                              RtlPrefetchMemoryNonTemporal,
 152                              Protect | PAGE_IS_WRITABLE);
 153             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 154             MmSetPageProtect(NULL, RtlPrefetchMemoryNonTemporal, Protect);
 155         }
 156     }
 157
 158     /* Check for, and enable SYSENTER support */
 159     KiRestoreFastSyscallReturnState();
 160
 161     /* Loop every CPU */
 162     i = KeActiveProcessors;
 163     for (Affinity = 1; i; Affinity <<= 1)
 164     {
 165         /* Check if this is part of the set */
 166         if (i & Affinity)
 167         {
 168             /* Run on this CPU */
 169             i &= ~Affinity;
 170             KeSetSystemAffinityThread(Affinity);
 171
 172             /* Reset MHz to 0 for this CPU */
 173             KeGetCurrentPrcb()->MHz = 0;
 174
 175             /* Check if we can use RDTSC */
 176             if (KeFeatureBits & KF_RDTSC)
 177             {
 178                 /* Start sampling loop */
 179                 for (;;)
 180                 {
 181                     /* Do a dummy CPUID to start the sample */
 182                     CPUID(Dummy, 0);
 183
 184                     /* Fill out the starting data */
 185                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 186                     CurrentSample->TSCStart = __rdtsc();
 187                     CurrentSample->PerfFreq.QuadPart = -50000;
 188
 189                     /* Sleep for this sample */
 190                     KeDelayExecutionThread(KernelMode,
 191                                            FALSE,
 192                                            &CurrentSample->PerfFreq);
 193
 194                     /* Do another dummy CPUID */
 195                     CPUID(Dummy, 0);
 196
 197                     /* Fill out the ending data */
 198                     CurrentSample->PerfEnd =
 199                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 200                     CurrentSample->TSCEnd = __rdtsc();
 201
 202                     /* Calculate the differences */
 203                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 204                                                CurrentSample->PerfStart.QuadPart;
 205                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 206                                               CurrentSample->TSCStart;
 207
 208                     /* Compute CPU Speed */
 209                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 210                                                   CurrentSample->
 211                                                   PerfFreq.QuadPart + 500000) /
 212                                                  (CurrentSample->PerfDelta *
 213                                                   1000000));
 214
 215                     /* Check if this isn't the first sample */
 216                     if (Sample)
 217                     {
 218                         /* Check if we got a good precision within 1MHz */
 219                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 220                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 221                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 222                         {
 223                             /* We did, stop sampling */
 224                             break;
 225                         }
 226                     }
 227
 228                     /* Move on */
 229                     CurrentSample++;
 230                     Sample++;
 231
 232                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 233                     {
 234                         /* Restart */
 235                         CurrentSample = Samples;
 236                         Sample = 0;
 237                     }
 238                 }
 239
 240                 /* Save the CPU Speed */
 241                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 242             }
 243
 244             /* Check if we have MTRR */
 245             if (KeFeatureBits & KF_MTRR)
 246             {
 247                 /* Then manually initialize MTRR for the CPU */
 248                 KiInitializeMTRR(i ? FALSE : TRUE);
 249             }
 250
 251             /* Check if we have AMD MTRR and initialize it for the CPU */
 252             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 253
 254             /* Check if this is a buggy Pentium and apply the fixup if so */
 255             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 256
 257             /* Check if the CPU supports FXSR */
 258             if (KeFeatureBits & KF_FXSR)
 259             {
 260                 /* Get the current thread NPX state */
 261                 FxSaveArea = (PVOID)
 262                              ((ULONG_PTR)KeGetCurrentThread()->InitialStack -
 263                              NPX_FRAME_LENGTH);
 264
 265                 /* Clear initial MXCsr mask */
 266                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 267
 268                 /* Save the current NPX State */
 269 #ifdef __GNUC__
 270                 asm volatile("fxsave %0\n\t" : "=m" (*FxSaveArea));
 271 #else
 272                 __asm fxsave [FxSaveArea]
 273 #endif
 274                 /* Check if the current mask doesn't match the reserved bits */
 275                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 276                 {
 277                     /* Then use whatever it's holding */
 278                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 279                 }
 280
 281                 /* Check if nobody set the kernel-wide mask */
 282                 if (!KiMXCsrMask)
 283                 {
 284                     /* Then use the one we calculated above */
 285                     KiMXCsrMask = MXCsrMask;
 286                 }
 287                 else
 288                 {
 289                     /* Was it set to the same value we found now? */
 290                     if (KiMXCsrMask != MXCsrMask)
 291                     {
 292                         /* No, something is definitely wrong */
 293                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 294                                      KF_FXSR,
 295                                      KiMXCsrMask,
 296                                      MXCsrMask,
 297                                      0);
 298                     }
 299                 }
 300
 301                 /* Now set the kernel mask */
 302                 KiMXCsrMask &= MXCsrMask;
 303             }
 304         }
 305     }
 306
 307     /* Return affinity back to where it was */
 308     KeRevertToUserAffinityThread();
 309
 310     /* NT allows limiting the duration of an ISR with a registry key */
 311     if (KiTimeLimitIsrMicroseconds)
 312     {
 313         /* FIXME: TODO */
 314         DPRINT1("ISR Time Limit not yet supported\n");
 315     }
 316 }
 317
 318 VOID
 319 NTAPI
 320 KiInitializePcr(IN ULONG ProcessorNumber,
 321                 IN PKIPCR Pcr,
 322                 IN PKIDTENTRY Idt,
 323                 IN PKGDTENTRY Gdt,
 324                 IN PKTSS Tss,
 325                 IN PKTHREAD IdleThread,
 326                 IN PVOID DpcStack)
 327 {
 328     /* Setup the TIB */
 329     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 330     Pcr->NtTib.StackBase = 0;
 331     Pcr->NtTib.StackLimit = 0;
 332     Pcr->NtTib.Self = NULL;
 333
 334     /* Set the Current Thread */
 335     Pcr->PrcbData.CurrentThread = IdleThread;
 336
 337     /* Set pointers to ourselves */
 338     Pcr->Self = (PKPCR)Pcr;
 339     Pcr->Prcb = &Pcr->PrcbData;
 340
 341     /* Set the PCR Version */
 342     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 343     Pcr->MinorVersion = PCR_MINOR_VERSION;
 344
 345     /* Set the PCRB Version */
 346     Pcr->PrcbData.MajorVersion = 1;
 347     Pcr->PrcbData.MinorVersion = 1;
 348
 349     /* Set the Build Type */
 350     Pcr->PrcbData.BuildType = 0;
 351 #ifndef CONFIG_SMP
 352     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 353 #endif
 354 #ifdef DBG
 355     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 356 #endif
 357
 358     /* Set the Processor Number and current Processor Mask */
 359     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 360     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 361
 362     /* Set the PRCB for this Processor */
 363     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 364
 365     /* Start us out at PASSIVE_LEVEL */
 366     Pcr->Irql = PASSIVE_LEVEL;
 367
 368     /* Set the GDI, IDT, TSS and DPC Stack */
 369     Pcr->GDT = (PVOID)Gdt;
 370     Pcr->IDT = Idt;
 371     Pcr->TSS = Tss;
 372     Pcr->TssCopy = Tss;
 373     Pcr->PrcbData.DpcStack = DpcStack;
 374
 375     /* Setup the processor set */
 376     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 377 }
 378
 379 VOID
 380 NTAPI
 381 KiInitializeKernel(IN PKPROCESS InitProcess,
 382                    IN PKTHREAD InitThread,
 383                    IN PVOID IdleStack,
 384                    IN PKPRCB Prcb,
 385                    IN CCHAR Number,
 386                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 387 {
 388     BOOLEAN NpxPresent;
 389     ULONG FeatureBits;
 390     ULONG PageDirectory[2];
 391     PVOID DpcStack;
 392     ULONG Vendor[3];
 393
 394     /* Detect and set the CPU Type */
 395     KiSetProcessorType();
 396
 397     /* Set CR0 features based on detected CPU */
 398     KiSetCR0Bits();
 399
 400     /* Check if an FPU is present */
 401     NpxPresent = KiIsNpxPresent();
 402
 403     /* Initialize the Power Management Support for this PRCB */
 404     PoInitializePrcb(Prcb);
 405
 406     /* Bugcheck if this is a 386 CPU */
 407     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 408
 409     /* Get the processor features for the CPU */
 410     FeatureBits = KiGetFeatureBits();
 411
 412     /* Set the default NX policy (opt-in) */
 413     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 414
 415     /* Check if NPX is always on */
 416     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 417     {
 418         /* Set it always on */
 419         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 420         FeatureBits |= KF_NX_ENABLED;
 421     }
 422     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 423     {
 424         /* Set it in opt-out mode */
 425         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 426         FeatureBits |= KF_NX_ENABLED;
 427     }
 428     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 429              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 430     {
 431         /* Set the feature bits */
 432         FeatureBits |= KF_NX_ENABLED;
 433     }
 434     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 435              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 436     {
 437         /* Set disabled mode */
 438         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 439         FeatureBits |= KF_NX_DISABLED;
 440     }
 441
 442     /* Save feature bits */
 443     Prcb->FeatureBits = FeatureBits;
 444
 445     /* Save CPU state */
 446     KiSaveProcessorControlState(&Prcb->ProcessorState);
 447
 448     /* Get cache line information for this CPU */
 449     KiGetCacheInformation();
 450
 451     /* Initialize spinlocks and DPC data */
 452     KiInitSpinLocks(Prcb, Number);
 453
 454     /* Check if this is the Boot CPU */
 455     if (!Number)
 456     {
 457         /* Set Node Data */
 458         KeNodeBlock[0] = &KiNode0;
 459         Prcb->ParentNode = KeNodeBlock[0];
 460         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 461
 462         /* Set boot-level flags */
 463         KeI386NpxPresent = NpxPresent;
 464         KeI386CpuType = Prcb->CpuType;
 465         KeI386CpuStep = Prcb->CpuStep;
 466         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 467         KeProcessorLevel = (USHORT)Prcb->CpuType;
 468         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 469         KeFeatureBits = FeatureBits;
 470         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 471         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 472
 473         /* Detect 8-byte compare exchange support */
 474         if (!(KeFeatureBits & KF_CMPXCHG8B))
 475         {
 476             /* Copy the vendor string */
 477             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 478
 479             /* Bugcheck the system. Windows *requires* this */
 480             KeBugCheckEx(0x5D,
 481                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 482                          Vendor[0],
 483                          Vendor[1],
 484                          Vendor[2]);
 485         }
 486
 487         /* Set the current MP Master KPRCB to the Boot PRCB */
 488         Prcb->MultiThreadSetMaster = Prcb;
 489
 490         /* Lower to APC_LEVEL */
 491         KeLowerIrql(APC_LEVEL);
 492
 493         /* Initialize some spinlocks */
 494         KeInitializeSpinLock(&KiFreezeExecutionLock);
 495         KeInitializeSpinLock(&Ki486CompatibilityLock);
 496
 497         /* Initialize portable parts of the OS */
 498         KiInitSystem();
 499
 500         /* Initialize the Idle Process and the Process Listhead */
 501         InitializeListHead(&KiProcessListHead);
 502         PageDirectory[0] = 0;
 503         PageDirectory[1] = 0;
 504         KeInitializeProcess(InitProcess,
 505                             0,
 506                             0xFFFFFFFF,
 507                             PageDirectory,
 508                             FALSE);
 509         InitProcess->QuantumReset = MAXCHAR;
 510     }
 511     else
 512     {
 513         /* FIXME */
 514         DPRINT1("SMP Boot support not yet present\n");
 515     }
 516
 517     /* Setup the Idle Thread */
 518     KeInitializeThread(InitProcess,
 519                        InitThread,
 520                        NULL,
 521                        NULL,
 522                        NULL,
 523                        NULL,
 524                        NULL,
 525                        IdleStack);
 526     InitThread->NextProcessor = Number;
 527     InitThread->Priority = HIGH_PRIORITY;
 528     InitThread->State = Running;
 529     InitThread->Affinity = 1 << Number;
 530     InitThread->WaitIrql = DISPATCH_LEVEL;
 531     InitProcess->ActiveProcessors = 1 << Number;
 532
 533     /* HACK for MmUpdatePageDir */
 534     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 535
 536     /* Set basic CPU Features that user mode can read */
 537     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 538         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 539     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 540         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 541     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 542         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 543     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 544         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 545     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 546         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 547     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 548         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 549
 550     /* Set up the thread-related fields in the PRCB */
 551     Prcb->CurrentThread = InitThread;
 552     Prcb->NextThread = NULL;
 553     Prcb->IdleThread = InitThread;
 554
 555     /* Initialize the Kernel Executive */
 556     ExpInitializeExecutive(Number, LoaderBlock);
 557
 558     /* Only do this on the boot CPU */
 559     if (!Number)
 560     {
 561         /* Calculate the time reciprocal */
 562         KiTimeIncrementReciprocal =
 563             KiComputeReciprocal(KeMaximumIncrement,
 564                                 &KiTimeIncrementShiftCount);
 565
 566         /* Update DPC Values in case they got updated by the executive */
 567         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 568         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 569         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 570
 571         /* Allocate the DPC Stack */
 572         DpcStack = MmCreateKernelStack(FALSE, 0);
 573         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 574         Prcb->DpcStack = DpcStack;
 575
 576         /* Allocate the IOPM save area. */
 577         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 578                                                   PAGE_SIZE * 2,
 579                                                   TAG('K', 'e', ' ', ' '));
 580         if (!Ki386IopmSaveArea)
 581         {
 582             /* Bugcheck. We need this for V86/VDM support. */
 583             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 584         }
 585     }
 586
 587     /* Raise to Dispatch */
 588     KfRaiseIrql(DISPATCH_LEVEL);
 589
 590     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 591     KeSetPriorityThread(InitThread, 0);
 592
 593     /* If there's no thread scheduled, put this CPU in the Idle summary */
 594     KiAcquirePrcbLock(Prcb);
 595     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 596     KiReleasePrcbLock(Prcb);
 597
 598     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 599     KfRaiseIrql(HIGH_LEVEL);
 600     LoaderBlock->Prcb = 0;
 601 }
 602
 603 VOID
 604 FASTCALL
 605 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 606                          IN PKIDTENTRY *Idt,
 607                          IN PKIPCR *Pcr,
 608                          IN PKTSS *Tss)
 609 {
 610     KDESCRIPTOR GdtDescriptor = { 0, 0, 0 }, IdtDescriptor = { 0, 0, 0 };
 611     KGDTENTRY TssSelector, PcrSelector;
 612     USHORT Tr = 0, Fs;
 613
 614     /* Get GDT and IDT descriptors */
 615     Ke386GetGlobalDescriptorTable(*(PKDESCRIPTOR)&GdtDescriptor.Limit);
 616     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 617
 618     /* Save IDT and GDT */
 619     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 620     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 621
 622     /* Get TSS and FS Selectors */
 623     Ke386GetTr(Tr);
 624     if (Tr != KGDT_TSS) Tr = KGDT_TSS; // FIXME: HACKHACK
 625     Fs = Ke386GetFs();
 626
 627     /* Get PCR Selector, mask it and get its GDT Entry */
 628     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 629
 630     /* Get the KPCR itself */
 631     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 632                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 633                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 634
 635     /* Get TSS Selector, mask it and get its GDT Entry */
 636     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 637
 638     /* Get the KTSS itself */
 639     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 640                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 641                               TssSelector.HighWord.Bytes.BaseHi << 24);
 642 }
 643
 644 VOID
 645 NTAPI
 646 KiSystemStartupReal(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 647 {
 648     ULONG Cpu;
 649     PKTHREAD InitialThread;
 650     ULONG InitialStack;
 651     PKGDTENTRY Gdt;
 652     PKIDTENTRY Idt;
 653     KIDTENTRY NmiEntry, DoubleFaultEntry;
 654     PKTSS Tss;
 655     PKIPCR Pcr;
 656
 657     /* Save the loader block and get the current CPU */
 658     KeLoaderBlock = LoaderBlock;
 659     Cpu = KeNumberProcessors;
 660     if (!Cpu)
 661     {
 662         /* If this is the boot CPU, set FS and the CPU Number*/
 663         Ke386SetFs(KGDT_R0_PCR);
 664         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 665
 666         /* Set the initial stack and idle thread as well */
 667         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 668         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 669     }
 670
 671     /* Save the initial thread and stack */
 672     InitialStack = LoaderBlock->KernelStack;
 673     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 674
 675     /* Clean the APC List Head */
 676     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 677
 678     /* Initialize the machine type */
 679     KiInitializeMachineType();
 680
 681     /* Skip initial setup if this isn't the Boot CPU */
 682     if (Cpu) goto AppCpuInit;
 683
 684     /* Get GDT, IDT, PCR and TSS pointers */
 685     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 686
 687     /* Setup the TSS descriptors and entries */
 688     Ki386InitializeTss(Tss, Idt, Gdt);
 689
 690     /* Initialize the PCR */
 691     RtlZeroMemory(Pcr, PAGE_SIZE);
 692     KiInitializePcr(Cpu,
 693                     Pcr,
 694                     Idt,
 695                     Gdt,
 696                     Tss,
 697                     InitialThread,
 698                     KiDoubleFaultStack);
 699
 700     /* Set us as the current process */
 701     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 702
 703     /* Clear DR6/7 to cleanup bootloader debugging */
 704     __writefsdword(KPCR_TEB, 0);
 705     __writefsdword(KPCR_DR6, 0);
 706     __writefsdword(KPCR_DR7, 0);
 707
 708     /* Setup the IDT */
 709     KeInitExceptions();
 710
 711     /* Load Ring 3 selectors for DS/ES */
 712     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 713     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 714
 715     /* Save NMI and double fault traps */
 716     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 717     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 718
 719     /* Copy kernel's trap handlers */
 720     RtlCopyMemory(Idt,
 721                   (PVOID)KiIdtDescriptor.Base,
 722                   KiIdtDescriptor.Limit + 1);
 723
 724     /* Restore NMI and double fault */
 725     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 726     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 727
 728 AppCpuInit:
 729     /* Loop until we can release the freeze lock */
 730     do
 731     {
 732         /* Loop until execution can continue */
 733         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 734     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 735
 736     /* Setup CPU-related fields */
 737     __writefsdword(KPCR_NUMBER, Cpu);
 738     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 739     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 740     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 741
 742     /* Initialize the Processor with HAL */
 743     HalInitializeProcessor(Cpu, KeLoaderBlock);
 744
 745     /* Set active processors */
 746     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 747     KeNumberProcessors++;
 748
 749     /* Check if this is the boot CPU */
 750     if (!Cpu)
 751     {
 752         /* Initialize debugging system */
 753         KdInitSystem(0, KeLoaderBlock);
 754
 755         /* Check for break-in */
 756         if (KdPollBreakIn()) DbgBreakPointWithStatus(1);
 757     }
 758
 759     /* Raise to HIGH_LEVEL */
 760     KfRaiseIrql(HIGH_LEVEL);
 761
 762     /* Align stack and make space for the trap frame and NPX frame */
 763     InitialStack &= ~(KTRAP_FRAME_ALIGN - 1);
 764
 765     /* Switch to new kernel stack and start kernel bootstrapping */
 766     KiSetupStackAndInitializeKernel(&KiInitialProcess.Pcb,
 767                                     InitialThread,
 768                                     (PVOID)InitialStack,
 769                                     (PKPRCB)__readfsdword(KPCR_PRCB),
 770                                     (CCHAR)Cpu,
 771                                     KeLoaderBlock);
 772 }