reactos/ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 /* GLOBALS *******************************************************************/
  16
  17 /* Spinlocks used only on X86 */
  18 KSPIN_LOCK KiFreezeExecutionLock;
  19 KSPIN_LOCK Ki486CompatibilityLock;
  20
  21 /* BIOS Memory Map. Not NTLDR-compliant yet */
  22 extern ULONG KeMemoryMapRangeCount;
  23 extern ADDRESS_RANGE KeMemoryMap[64];
  24
  25 /* FUNCTIONS *****************************************************************/
  26
  27 VOID
  28 NTAPI
  29 KiInitMachineDependent(VOID)
  30 {
  31     ULONG Protect;
  32     ULONG CpuCount;
  33     BOOLEAN FbCaching = FALSE;
  34     NTSTATUS Status;
  35     ULONG ReturnLength;
  36     ULONG i, Affinity, Sample = 0;
  37     PFX_SAVE_AREA FxSaveArea;
  38     ULONG MXCsrMask = 0xFFBF;
  39     ULONG Dummy[4];
  40     KI_SAMPLE_MAP Samples[4];
  41     PKI_SAMPLE_MAP CurrentSample = Samples;
  42
  43     /* Check for large page support */
  44     if (KeFeatureBits & KF_LARGE_PAGE)
  45     {
  46         /* FIXME: Support this */
  47         DPRINT1("Large Page support detected but not yet taken advantage of!\n");
  48     }
  49
  50     /* Check for global page support */
  51     if (KeFeatureBits & KF_GLOBAL_PAGE)
  52     {
  53         /* Do an IPI to enable it on all CPUs */
  54         CpuCount = KeNumberProcessors;
  55         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  56     }
  57
  58     /* Check for PAT and/or MTRR support */
  59     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  60     {
  61         /* Query the HAL to make sure we can use it */
  62         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  63                                            sizeof(BOOLEAN),
  64                                            &FbCaching,
  65                                            &ReturnLength);
  66         if ((NT_SUCCESS(Status)) && (FbCaching))
  67         {
  68             /* We can't, disable it */
  69             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  70         }
  71     }
  72
  73     /* Check for PAT support and enable it */
  74     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  75
  76     /* Assume no errata for now */
  77     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  78
  79     /* Check if we have an NPX */
  80     if (KeI386NpxPresent)
  81     {
  82         /* Loop every CPU */
  83         i = KeActiveProcessors;
  84         for (Affinity = 1; i; Affinity <<= 1)
  85         {
  86             /* Check if this is part of the set */
  87             if (i & Affinity)
  88             {
  89                 /* Run on this CPU */
  90                 i &= ~Affinity;
  91                 KeSetSystemAffinityThread(Affinity);
  92
  93                 /* Detect FPU errata */
  94                 if (KiIsNpxErrataPresent())
  95                 {
  96                     /* Disable NPX support */
  97                     KeI386NpxPresent = FALSE;
  98                     SharedUserData->
  99                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 100                         TRUE;
 101                     break;
 102                 }
 103             }
 104         }
 105     }
 106
 107     /* If there's no NPX, then we're emulating the FPU */
 108     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 109         !KeI386NpxPresent;
 110
 111     /* Check if there's no NPX, so that we can disable associated features */
 112     if (!KeI386NpxPresent)
 113     {
 114         /* Remove NPX-related bits */
 115         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 116
 117         /* Disable kernel flags */
 118         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 119
 120         /* Disable processor features that might've been set until now */
 121         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 122         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 123         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 124         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 125         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 126     }
 127
 128     /* Check for CR4 support */
 129     if (KeFeatureBits & KF_CR4)
 130     {
 131         /* Do an IPI call to enable the Debug Exceptions */
 132         CpuCount = KeNumberProcessors;
 133         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 134     }
 135
 136     /* Check if FXSR was found */
 137     if (KeFeatureBits & KF_FXSR)
 138     {
 139         /* Do an IPI call to enable the FXSR */
 140         CpuCount = KeNumberProcessors;
 141         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 142
 143         /* Check if XMM was found too */
 144         if (KeFeatureBits & KF_XMMI)
 145         {
 146             /* Do an IPI call to enable XMMI exceptions */
 147             CpuCount = KeNumberProcessors;
 148             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 149
 150             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 151
 152             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 153             Protect = MmGetPageProtect(NULL, RtlPrefetchMemoryNonTemporal);
 154             MmSetPageProtect(NULL,
 155                              RtlPrefetchMemoryNonTemporal,
 156                              Protect | PAGE_IS_WRITABLE);
 157             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 158             MmSetPageProtect(NULL, RtlPrefetchMemoryNonTemporal, Protect);
 159         }
 160     }
 161
 162     /* Check for, and enable SYSENTER support */
 163     KiRestoreFastSyscallReturnState();
 164
 165     /* Loop every CPU */
 166     i = KeActiveProcessors;
 167     for (Affinity = 1; i; Affinity <<= 1)
 168     {
 169         /* Check if this is part of the set */
 170         if (i & Affinity)
 171         {
 172             /* Run on this CPU */
 173             i &= ~Affinity;
 174             KeSetSystemAffinityThread(Affinity);
 175
 176             /* Reset MHz to 0 for this CPU */
 177             KeGetCurrentPrcb()->MHz = 0;
 178
 179             /* Check if we can use RDTSC */
 180             if (KeFeatureBits & KF_RDTSC)
 181             {
 182                 /* Start sampling loop */
 183                 for (;;)
 184                 {
 185                     /* Do a dummy CPUID to start the sample */
 186                     CPUID(Dummy, 0);
 187
 188                     /* Fill out the starting data */
 189                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 190                     CurrentSample->TSCStart = __rdtsc();
 191                     CurrentSample->PerfFreq.QuadPart = -50000;
 192
 193                     /* Sleep for this sample */
 194                     KeDelayExecutionThread(KernelMode,
 195                                            FALSE,
 196                                            &CurrentSample->PerfFreq);
 197
 198                     /* Do another dummy CPUID */
 199                     CPUID(Dummy, 0);
 200
 201                     /* Fill out the ending data */
 202                     CurrentSample->PerfEnd =
 203                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 204                     CurrentSample->TSCEnd = __rdtsc();
 205
 206                     /* Calculate the differences */
 207                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 208                                                CurrentSample->PerfStart.QuadPart;
 209                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 210                                               CurrentSample->TSCStart;
 211
 212                     /* Compute CPU Speed */
 213                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 214                                                   CurrentSample->
 215                                                   PerfFreq.QuadPart + 500000) /
 216                                                  (CurrentSample->PerfDelta *
 217                                                   1000000));
 218
 219                     /* Check if this isn't the first sample */
 220                     if (Sample)
 221                     {
 222                         /* Check if we got a good precision within 1MHz */
 223                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 224                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 225                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 226                         {
 227                             /* We did, stop sampling */
 228                             break;
 229                         }
 230                     }
 231
 232                     /* Move on */
 233                     CurrentSample++;
 234                     Sample++;
 235
 236                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 237                     {
 238                         /* Restart */
 239                         CurrentSample = Samples;
 240                         Sample = 0;
 241                     }
 242                 }
 243
 244                 /* Save the CPU Speed */
 245                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 246             }
 247
 248             /* Check if we have MTRR */
 249             if (KeFeatureBits & KF_MTRR)
 250             {
 251                 /* Then manually initialize MTRR for the CPU */
 252                 KiInitializeMTRR(i ? FALSE : TRUE);
 253             }
 254
 255             /* Check if we have AMD MTRR and initialize it for the CPU */
 256             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 257
 258             /* Check if this is a buggy Pentium and apply the fixup if so */
 259             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 260
 261             /* Check if the CPU supports FXSR */
 262             if (KeFeatureBits & KF_FXSR)
 263             {
 264                 /* Get the current thread NPX state */
 265                 FxSaveArea = (PVOID)
 266                              ((ULONG_PTR)KeGetCurrentThread()->InitialStack -
 267                              NPX_FRAME_LENGTH);
 268
 269                 /* Clear initial MXCsr mask */
 270                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 271
 272                 /* Save the current NPX State */
 273 #ifdef __GNUC__
 274                 asm volatile("fxsave %0\n\t" : "=m" (*FxSaveArea));
 275 #else
 276                 __asm fxsave [FxSaveArea]
 277 #endif
 278                 /* Check if the current mask doesn't match the reserved bits */
 279                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 280                 {
 281                     /* Then use whatever it's holding */
 282                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 283                 }
 284
 285                 /* Check if nobody set the kernel-wide mask */
 286                 if (!KiMXCsrMask)
 287                 {
 288                     /* Then use the one we calculated above */
 289                     KiMXCsrMask = MXCsrMask;
 290                 }
 291                 else
 292                 {
 293                     /* Was it set to the same value we found now? */
 294                     if (KiMXCsrMask != MXCsrMask)
 295                     {
 296                         /* No, something is definitely wrong */
 297                         KEBUGCHECKEX(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 298                                      KF_FXSR,
 299                                      KiMXCsrMask,
 300                                      MXCsrMask,
 301                                      0);
 302                     }
 303                 }
 304
 305                 /* Now set the kernel mask */
 306                 KiMXCsrMask &= MXCsrMask;
 307             }
 308         }
 309     }
 310
 311     /* Return affinity back to where it was */
 312     KeRevertToUserAffinityThread();
 313
 314     /* NT allows limiting the duration of an ISR with a registry key */
 315     if (KiTimeLimitIsrMicroseconds)
 316     {
 317         /* FIXME: TODO */
 318         DPRINT1("ISR Time Limit not yet supported\n");
 319     }
 320 }
 321
 322 VOID
 323 NTAPI
 324 KiInitializePcr(IN ULONG ProcessorNumber,
 325                 IN PKIPCR Pcr,
 326                 IN PKIDTENTRY Idt,
 327                 IN PKGDTENTRY Gdt,
 328                 IN PKTSS Tss,
 329                 IN PKTHREAD IdleThread,
 330                 IN PVOID DpcStack)
 331 {
 332     /* Setup the TIB */
 333     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 334     Pcr->NtTib.StackBase = 0;
 335     Pcr->NtTib.StackLimit = 0;
 336     Pcr->NtTib.Self = NULL;
 337
 338     /* Set the Current Thread */
 339     Pcr->PrcbData.CurrentThread = IdleThread;
 340
 341     /* Set pointers to ourselves */
 342     Pcr->Self = (PKPCR)Pcr;
 343     Pcr->Prcb = &Pcr->PrcbData;
 344
 345     /* Set the PCR Version */
 346     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 347     Pcr->MinorVersion = PCR_MINOR_VERSION;
 348
 349     /* Set the PCRB Version */
 350     Pcr->PrcbData.MajorVersion = 1;
 351     Pcr->PrcbData.MinorVersion = 1;
 352
 353     /* Set the Build Type */
 354     Pcr->PrcbData.BuildType = 0;
 355 #ifndef CONFIG_SMP
 356     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 357 #endif
 358 #ifdef DBG
 359     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 360 #endif
 361
 362     /* Set the Processor Number and current Processor Mask */
 363     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 364     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 365
 366     /* Set the PRCB for this Processor */
 367     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 368
 369     /* Start us out at PASSIVE_LEVEL */
 370     Pcr->Irql = PASSIVE_LEVEL;
 371
 372     /* Set the GDI, IDT, TSS and DPC Stack */
 373     Pcr->GDT = (PVOID)Gdt;
 374     Pcr->IDT = Idt;
 375     Pcr->TSS = Tss;
 376     Pcr->TssCopy = Tss;
 377     Pcr->PrcbData.DpcStack = DpcStack;
 378
 379     /* Setup the processor set */
 380     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 381 }
 382
 383 VOID
 384 NTAPI
 385 KiInitializeKernel(IN PKPROCESS InitProcess,
 386                    IN PKTHREAD InitThread,
 387                    IN PVOID IdleStack,
 388                    IN PKPRCB Prcb,
 389                    IN CCHAR Number,
 390                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 391 {
 392     BOOLEAN NpxPresent;
 393     ULONG FeatureBits;
 394     LARGE_INTEGER PageDirectory;
 395     PVOID DpcStack;
 396     ULONG Vendor[3];
 397
 398     /* Detect and set the CPU Type */
 399     KiSetProcessorType();
 400
 401     /* Set CR0 features based on detected CPU */
 402     KiSetCR0Bits();
 403
 404     /* Check if an FPU is present */
 405     NpxPresent = KiIsNpxPresent();
 406
 407     /* Initialize the Power Management Support for this PRCB */
 408     PoInitializePrcb(Prcb);
 409
 410     /* Bugcheck if this is a 386 CPU */
 411     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 412
 413     /* Get the processor features for the CPU */
 414     FeatureBits = KiGetFeatureBits();
 415
 416     /* Set the default NX policy (opt-in) */
 417     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 418
 419     /* Check if NPX is always on */
 420     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 421     {
 422         /* Set it always on */
 423         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 424         FeatureBits |= KF_NX_ENABLED;
 425     }
 426     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 427     {
 428         /* Set it in opt-out mode */
 429         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 430         FeatureBits |= KF_NX_ENABLED;
 431     }
 432     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 433              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 434     {
 435         /* Set the feature bits */
 436         FeatureBits |= KF_NX_ENABLED;
 437     }
 438     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 439              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 440     {
 441         /* Set disabled mode */
 442         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 443         FeatureBits |= KF_NX_DISABLED;
 444     }
 445
 446     /* Save feature bits */
 447     Prcb->FeatureBits = FeatureBits;
 448
 449     /* Save CPU state */
 450     KiSaveProcessorControlState(&Prcb->ProcessorState);
 451
 452     /* Get cache line information for this CPU */
 453     KiGetCacheInformation();
 454
 455     /* Initialize spinlocks and DPC data */
 456     KiInitSpinLocks(Prcb, Number);
 457
 458     /* Check if this is the Boot CPU */
 459     if (!Number)
 460     {
 461         /* Set Node Data */
 462         KeNodeBlock[0] = &KiNode0;
 463         Prcb->ParentNode = KeNodeBlock[0];
 464         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 465
 466         /* Set boot-level flags */
 467         KeI386NpxPresent = NpxPresent;
 468         KeI386CpuType = Prcb->CpuType;
 469         KeI386CpuStep = Prcb->CpuStep;
 470         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 471         KeProcessorLevel = (USHORT)Prcb->CpuType;
 472         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 473         KeFeatureBits = FeatureBits;
 474         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 475         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 476
 477         /* Detect 8-byte compare exchange support */
 478         if (!(KeFeatureBits & KF_CMPXCHG8B))
 479         {
 480             /* Copy the vendor string */
 481             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 482
 483             /* Bugcheck the system. Windows *requires* this */
 484             KeBugCheckEx(0x5D,
 485                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 486                          Vendor[0],
 487                          Vendor[1],
 488                          Vendor[2]);
 489         }
 490
 491         /* Set the current MP Master KPRCB to the Boot PRCB */
 492         Prcb->MultiThreadSetMaster = Prcb;
 493
 494         /* Lower to APC_LEVEL */
 495         KeLowerIrql(APC_LEVEL);
 496
 497         /* Initialize some spinlocks */
 498         KeInitializeSpinLock(&KiFreezeExecutionLock);
 499         KeInitializeSpinLock(&Ki486CompatibilityLock);
 500
 501         /* Initialize portable parts of the OS */
 502         KiInitSystem();
 503
 504         /* Initialize the Idle Process and the Process Listhead */
 505         InitializeListHead(&KiProcessListHead);
 506         PageDirectory.QuadPart = 0;
 507         KeInitializeProcess(InitProcess,
 508                             0,
 509                             0xFFFFFFFF,
 510                             &PageDirectory,
 511                             FALSE);
 512         InitProcess->QuantumReset = MAXCHAR;
 513     }
 514     else
 515     {
 516         /* FIXME */
 517         DPRINT1("SMP Boot support not yet present\n");
 518     }
 519
 520     /* Setup the Idle Thread */
 521     KeInitializeThread(InitProcess,
 522                        InitThread,
 523                        NULL,
 524                        NULL,
 525                        NULL,
 526                        NULL,
 527                        NULL,
 528                        IdleStack);
 529     InitThread->NextProcessor = Number;
 530     InitThread->Priority = HIGH_PRIORITY;
 531     InitThread->State = Running;
 532     InitThread->Affinity = 1 << Number;
 533     InitThread->WaitIrql = DISPATCH_LEVEL;
 534     InitProcess->ActiveProcessors = 1 << Number;
 535
 536     /* HACK for MmUpdatePageDir */
 537     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 538
 539     /* Set basic CPU Features that user mode can read */
 540     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 541         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 542     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 543         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 544     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 545         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 546     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 547         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 548     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 549         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 550     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 551         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 552
 553     /* Set up the thread-related fields in the PRCB */
 554     Prcb->CurrentThread = InitThread;
 555     Prcb->NextThread = NULL;
 556     Prcb->IdleThread = InitThread;
 557
 558     /* Initialize the Kernel Executive */
 559     ExpInitializeExecutive(Number, LoaderBlock);
 560
 561     /* Only do this on the boot CPU */
 562     if (!Number)
 563     {
 564         /* Calculate the time reciprocal */
 565         KiTimeIncrementReciprocal =
 566             KiComputeReciprocal(KeMaximumIncrement,
 567                                 &KiTimeIncrementShiftCount);
 568
 569         /* Update DPC Values in case they got updated by the executive */
 570         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 571         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 572         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 573
 574         /* Allocate the DPC Stack */
 575         DpcStack = MmCreateKernelStack(FALSE, 0);
 576         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 577         Prcb->DpcStack = DpcStack;
 578
 579         /* Allocate the IOPM save area. */
 580         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 581                                                   PAGE_SIZE * 2,
 582                                                   TAG('K', 'e', ' ', ' '));
 583         if (!Ki386IopmSaveArea)
 584         {
 585             /* Bugcheck. We need this for V86/VDM support. */
 586             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 587         }
 588     }
 589
 590     /* Raise to Dispatch */
 591     KfRaiseIrql(DISPATCH_LEVEL);
 592
 593     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 594     KeSetPriorityThread(InitThread, 0);
 595
 596     /* If there's no thread scheduled, put this CPU in the Idle summary */
 597     KiAcquirePrcbLock(Prcb);
 598     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 599     KiReleasePrcbLock(Prcb);
 600
 601     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 602     KfRaiseIrql(HIGH_LEVEL);
 603     LoaderBlock->Prcb = 0;
 604 }
 605
 606 VOID
 607 FASTCALL
 608 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 609                          IN PKIDTENTRY *Idt,
 610                          IN PKIPCR *Pcr,
 611                          IN PKTSS *Tss)
 612 {
 613     KDESCRIPTOR GdtDescriptor = {0}, IdtDescriptor = {0};
 614     KGDTENTRY TssSelector, PcrSelector;
 615     USHORT Tr = 0, Fs;
 616
 617     /* Get GDT and IDT descriptors */
 618     Ke386GetGlobalDescriptorTable(*(PKDESCRIPTOR)&GdtDescriptor.Limit);
 619     Ke386GetInterruptDescriptorTable(*(PKDESCRIPTOR)&IdtDescriptor.Limit);
 620
 621     /* Save IDT and GDT */
 622     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 623     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 624
 625     /* Get TSS and FS Selectors */
 626     Ke386GetTr(Tr);
 627     if (Tr != KGDT_TSS) Tr = KGDT_TSS; // FIXME: HACKHACK
 628     Fs = Ke386GetFs();
 629
 630     /* Get PCR Selector, mask it and get its GDT Entry */
 631     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 632
 633     /* Get the KPCR itself */
 634     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 635                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 636                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 637
 638     /* Get TSS Selector, mask it and get its GDT Entry */
 639     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 640
 641     /* Get the KTSS itself */
 642     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 643                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 644                               TssSelector.HighWord.Bytes.BaseHi << 24);
 645 }
 646
 647 VOID
 648 NTAPI
 649 KiSystemStartupReal(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 650 {
 651     ULONG Cpu;
 652     PKTHREAD InitialThread;
 653     ULONG InitialStack;
 654     PKGDTENTRY Gdt;
 655     PKIDTENTRY Idt;
 656     KIDTENTRY NmiEntry, DoubleFaultEntry;
 657     PKTSS Tss;
 658     PKIPCR Pcr;
 659
 660     /* Save the loader block and get the current CPU */
 661     KeLoaderBlock = LoaderBlock;
 662     Cpu = KeNumberProcessors;
 663     if (!Cpu)
 664     {
 665         /* If this is the boot CPU, set FS and the CPU Number*/
 666         Ke386SetFs(KGDT_R0_PCR);
 667         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 668
 669         /* Set the initial stack and idle thread as well */
 670         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 671         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 672     }
 673
 674     /* Save the initial thread and stack */
 675     InitialStack = LoaderBlock->KernelStack;
 676     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 677
 678     /* Clean the APC List Head */
 679     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 680
 681     /* Initialize the machine type */
 682     KiInitializeMachineType();
 683
 684     /* Skip initial setup if this isn't the Boot CPU */
 685     if (Cpu) goto AppCpuInit;
 686
 687     /* Get GDT, IDT, PCR and TSS pointers */
 688     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 689
 690     /* Setup the TSS descriptors and entries */
 691     Ki386InitializeTss(Tss, Idt, Gdt);
 692
 693     /* Initialize the PCR */
 694     RtlZeroMemory(Pcr, PAGE_SIZE);
 695     KiInitializePcr(Cpu,
 696                     Pcr,
 697                     Idt,
 698                     Gdt,
 699                     Tss,
 700                     InitialThread,
 701                     KiDoubleFaultStack);
 702
 703     /* Set us as the current process */
 704     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 705
 706     /* Clear DR6/7 to cleanup bootloader debugging */
 707     __writefsdword(KPCR_TEB, 0);
 708     __writefsdword(KPCR_DR6, 0);
 709     __writefsdword(KPCR_DR7, 0);
 710
 711     /* Setup the IDT */
 712     KeInitExceptions();
 713
 714     /* Load Ring 3 selectors for DS/ES */
 715     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 716     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 717
 718     /* Save NMI and double fault traps */
 719     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 720     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 721
 722     /* Copy kernel's trap handlers */
 723     RtlCopyMemory(Idt,
 724                   (PVOID)KiIdtDescriptor.Base,
 725                   KiIdtDescriptor.Limit + 1);
 726
 727     /* Restore NMI and double fault */
 728     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 729     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 730
 731 AppCpuInit:
 732     /* Loop until we can release the freeze lock */
 733     do
 734     {
 735         /* Loop until execution can continue */
 736         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 737     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 738
 739     /* Setup CPU-related fields */
 740     __writefsdword(KPCR_NUMBER, Cpu);
 741     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 742     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 743     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 744
 745     /* Initialize the Processor with HAL */
 746     HalInitializeProcessor(Cpu, KeLoaderBlock);
 747
 748     /* Set active processors */
 749     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 750     KeNumberProcessors++;
 751
 752     /* Check if this is the boot CPU */
 753     if (!Cpu)
 754     {
 755         /* Initialize debugging system */
 756         KdInitSystem(0, KeLoaderBlock);
 757
 758         /* Check for break-in */
 759         if (KdPollBreakIn()) DbgBreakPointWithStatus(1);
 760     }
 761
 762     /* Raise to HIGH_LEVEL */
 763     KfRaiseIrql(HIGH_LEVEL);
 764
 765     /* Align stack and make space for the trap frame and NPX frame */
 766     InitialStack &= ~(KTRAP_FRAME_ALIGN - 1);
 767
 768     /* Switch to new kernel stack and start kernel bootstrapping */
 769     KiSetupStackAndInitializeKernel(&KiInitialProcess.Pcb,
 770                                     InitialThread,
 771                                     (PVOID)InitialStack,
 772                                     (PKPRCB)__readfsdword(KPCR_PRCB),
 773                                     (CCHAR)Cpu,
 774                                     KeLoaderBlock);
 775 }