reactos/ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR P0BootStackData[KERNEL_STACK_SIZE] __attribute__((aligned (16))) = {0};
  20 UCHAR KiDoubleFaultStackData[KERNEL_STACK_SIZE] __attribute__((aligned (16))) = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* FUNCTIONS *****************************************************************/
  29
  30 VOID
  31 NTAPI
  32 KiInitMachineDependent(VOID)
  33 {
  34     ULONG Protect;
  35     ULONG CpuCount;
  36     BOOLEAN FbCaching = FALSE;
  37     NTSTATUS Status;
  38     ULONG ReturnLength;
  39     ULONG i, Affinity, Sample = 0;
  40     PFX_SAVE_AREA FxSaveArea;
  41     ULONG MXCsrMask = 0xFFBF;
  42     ULONG Dummy;
  43     KI_SAMPLE_MAP Samples[4];
  44     PKI_SAMPLE_MAP CurrentSample = Samples;
  45
  46     /* Check for large page support */
  47     if (KeFeatureBits & KF_LARGE_PAGE)
  48     {
  49         /* FIXME: Support this */
  50         DPRINT1("Large Page support detected but not yet taken advantage of\n");
  51     }
  52
  53     /* Check for global page support */
  54     if (KeFeatureBits & KF_GLOBAL_PAGE)
  55     {
  56         /* Do an IPI to enable it on all CPUs */
  57         CpuCount = KeNumberProcessors;
  58         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  59     }
  60
  61     /* Check for PAT and/or MTRR support */
  62     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  63     {
  64         /* Query the HAL to make sure we can use it */
  65         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  66                                            sizeof(BOOLEAN),
  67                                            &FbCaching,
  68                                            &ReturnLength);
  69         if ((NT_SUCCESS(Status)) && (FbCaching))
  70         {
  71             /* We can't, disable it */
  72             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  73         }
  74     }
  75
  76     /* Check for PAT support and enable it */
  77     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  78
  79     /* Assume no errata for now */
  80     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  81
  82     /* Check if we have an NPX */
  83     if (KeI386NpxPresent)
  84     {
  85         /* Loop every CPU */
  86         i = KeActiveProcessors;
  87         for (Affinity = 1; i; Affinity <<= 1)
  88         {
  89             /* Check if this is part of the set */
  90             if (i & Affinity)
  91             {
  92                 /* Run on this CPU */
  93                 i &= ~Affinity;
  94                 KeSetSystemAffinityThread(Affinity);
  95
  96                 /* Detect FPU errata */
  97                 if (KiIsNpxErrataPresent())
  98                 {
  99                     /* Disable NPX support */
 100                     KeI386NpxPresent = FALSE;
 101                     SharedUserData->
 102                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 103                         TRUE;
 104                     break;
 105                 }
 106             }
 107         }
 108     }
 109
 110     /* If there's no NPX, then we're emulating the FPU */
 111     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 112         !KeI386NpxPresent;
 113
 114     /* Check if there's no NPX, so that we can disable associated features */
 115     if (!KeI386NpxPresent)
 116     {
 117         /* Remove NPX-related bits */
 118         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 119
 120         /* Disable kernel flags */
 121         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 122
 123         /* Disable processor features that might've been set until now */
 124         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 125         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 126         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 127         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 128         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 129     }
 130
 131     /* Check for CR4 support */
 132     if (KeFeatureBits & KF_CR4)
 133     {
 134         /* Do an IPI call to enable the Debug Exceptions */
 135         CpuCount = KeNumberProcessors;
 136         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 137     }
 138
 139     /* Check if FXSR was found */
 140     if (KeFeatureBits & KF_FXSR)
 141     {
 142         /* Do an IPI call to enable the FXSR */
 143         CpuCount = KeNumberProcessors;
 144         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 145
 146         /* Check if XMM was found too */
 147         if (KeFeatureBits & KF_XMMI)
 148         {
 149             /* Do an IPI call to enable XMMI exceptions */
 150             CpuCount = KeNumberProcessors;
 151             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 152
 153             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 154
 155             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 156             Protect = MmGetPageProtect(NULL, RtlPrefetchMemoryNonTemporal);
 157             MmSetPageProtect(NULL,
 158                              RtlPrefetchMemoryNonTemporal,
 159                              Protect | PAGE_IS_WRITABLE);
 160             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 161             MmSetPageProtect(NULL, RtlPrefetchMemoryNonTemporal, Protect);
 162         }
 163     }
 164
 165     /* Check for, and enable SYSENTER support */
 166     KiRestoreFastSyscallReturnState();
 167
 168     /* Loop every CPU */
 169     i = KeActiveProcessors;
 170     for (Affinity = 1; i; Affinity <<= 1)
 171     {
 172         /* Check if this is part of the set */
 173         if (i & Affinity)
 174         {
 175             /* Run on this CPU */
 176             i &= ~Affinity;
 177             KeSetSystemAffinityThread(Affinity);
 178
 179             /* Reset MHz to 0 for this CPU */
 180             KeGetCurrentPrcb()->MHz = 0;
 181
 182             /* Check if we can use RDTSC */
 183             if (KeFeatureBits & KF_RDTSC)
 184             {
 185                 /* Start sampling loop */
 186                 for (;;)
 187                 {
 188                     /* Do a dummy CPUID to start the sample */
 189                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 190
 191                     /* Fill out the starting data */
 192                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 193                     CurrentSample->TSCStart = __rdtsc();
 194                     CurrentSample->PerfFreq.QuadPart = -50000;
 195
 196                     /* Sleep for this sample */
 197                     KeDelayExecutionThread(KernelMode,
 198                                            FALSE,
 199                                            &CurrentSample->PerfFreq);
 200
 201                     /* Do another dummy CPUID */
 202                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 203
 204                     /* Fill out the ending data */
 205                     CurrentSample->PerfEnd =
 206                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 207                     CurrentSample->TSCEnd = __rdtsc();
 208
 209                     /* Calculate the differences */
 210                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 211                                                CurrentSample->PerfStart.QuadPart;
 212                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 213                                               CurrentSample->TSCStart;
 214
 215                     /* Compute CPU Speed */
 216                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 217                                                   CurrentSample->
 218                                                   PerfFreq.QuadPart + 500000) /
 219                                                  (CurrentSample->PerfDelta *
 220                                                   1000000));
 221
 222                     /* Check if this isn't the first sample */
 223                     if (Sample)
 224                     {
 225                         /* Check if we got a good precision within 1MHz */
 226                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 227                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 228                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 229                         {
 230                             /* We did, stop sampling */
 231                             break;
 232                         }
 233                     }
 234
 235                     /* Move on */
 236                     CurrentSample++;
 237                     Sample++;
 238
 239                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 240                     {
 241                         /* Restart */
 242                         CurrentSample = Samples;
 243                         Sample = 0;
 244                     }
 245                 }
 246
 247                 /* Save the CPU Speed */
 248                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 249             }
 250
 251             /* Check if we have MTRR */
 252             if (KeFeatureBits & KF_MTRR)
 253             {
 254                 /* Then manually initialize MTRR for the CPU */
 255                 KiInitializeMTRR(i ? FALSE : TRUE);
 256             }
 257
 258             /* Check if we have AMD MTRR and initialize it for the CPU */
 259             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 260
 261             /* Check if this is a buggy Pentium and apply the fixup if so */
 262             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 263
 264             /* Check if the CPU supports FXSR */
 265             if (KeFeatureBits & KF_FXSR)
 266             {
 267                 /* Get the current thread NPX state */
 268                 FxSaveArea = (PVOID)
 269                              ((ULONG_PTR)KeGetCurrentThread()->InitialStack -
 270                              NPX_FRAME_LENGTH);
 271
 272                 /* Clear initial MXCsr mask */
 273                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 274
 275                 /* Save the current NPX State */
 276 #ifdef __GNUC__
 277                 asm volatile("fxsave %0\n\t" : "=m" (*FxSaveArea));
 278 #else
 279                 __asm fxsave [FxSaveArea]
 280 #endif
 281                 /* Check if the current mask doesn't match the reserved bits */
 282                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 283                 {
 284                     /* Then use whatever it's holding */
 285                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 286                 }
 287
 288                 /* Check if nobody set the kernel-wide mask */
 289                 if (!KiMXCsrMask)
 290                 {
 291                     /* Then use the one we calculated above */
 292                     KiMXCsrMask = MXCsrMask;
 293                 }
 294                 else
 295                 {
 296                     /* Was it set to the same value we found now? */
 297                     if (KiMXCsrMask != MXCsrMask)
 298                     {
 299                         /* No, something is definitely wrong */
 300                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 301                                      KF_FXSR,
 302                                      KiMXCsrMask,
 303                                      MXCsrMask,
 304                                      0);
 305                     }
 306                 }
 307
 308                 /* Now set the kernel mask */
 309                 KiMXCsrMask &= MXCsrMask;
 310             }
 311         }
 312     }
 313
 314     /* Return affinity back to where it was */
 315     KeRevertToUserAffinityThread();
 316
 317     /* NT allows limiting the duration of an ISR with a registry key */
 318     if (KiTimeLimitIsrMicroseconds)
 319     {
 320         /* FIXME: TODO */
 321         DPRINT1("ISR Time Limit not yet supported\n");
 322     }
 323 }
 324
 325 VOID
 326 NTAPI
 327 KiInitializePcr(IN ULONG ProcessorNumber,
 328                 IN PKIPCR Pcr,
 329                 IN PKIDTENTRY Idt,
 330                 IN PKGDTENTRY Gdt,
 331                 IN PKTSS Tss,
 332                 IN PKTHREAD IdleThread,
 333                 IN PVOID DpcStack)
 334 {
 335     /* Setup the TIB */
 336     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 337     Pcr->NtTib.StackBase = 0;
 338     Pcr->NtTib.StackLimit = 0;
 339     Pcr->NtTib.Self = NULL;
 340
 341     /* Set the Current Thread */
 342     Pcr->PrcbData.CurrentThread = IdleThread;
 343
 344     /* Set pointers to ourselves */
 345     Pcr->Self = (PKPCR)Pcr;
 346     Pcr->Prcb = &Pcr->PrcbData;
 347
 348     /* Set the PCR Version */
 349     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 350     Pcr->MinorVersion = PCR_MINOR_VERSION;
 351
 352     /* Set the PCRB Version */
 353     Pcr->PrcbData.MajorVersion = 1;
 354     Pcr->PrcbData.MinorVersion = 1;
 355
 356     /* Set the Build Type */
 357     Pcr->PrcbData.BuildType = 0;
 358 #ifndef CONFIG_SMP
 359     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 360 #endif
 361 #if DBG
 362     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 363 #endif
 364
 365     /* Set the Processor Number and current Processor Mask */
 366     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 367     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 368
 369     /* Set the PRCB for this Processor */
 370     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 371
 372     /* Start us out at PASSIVE_LEVEL */
 373     Pcr->Irql = PASSIVE_LEVEL;
 374
 375     /* Set the GDI, IDT, TSS and DPC Stack */
 376     Pcr->GDT = (PVOID)Gdt;
 377     Pcr->IDT = Idt;
 378     Pcr->TSS = Tss;
 379     Pcr->TssCopy = Tss;
 380     Pcr->PrcbData.DpcStack = DpcStack;
 381
 382     /* Setup the processor set */
 383     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 384 }
 385
 386 VOID
 387 NTAPI
 388 KiInitializeKernel(IN PKPROCESS InitProcess,
 389                    IN PKTHREAD InitThread,
 390                    IN PVOID IdleStack,
 391                    IN PKPRCB Prcb,
 392                    IN CCHAR Number,
 393                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 394 {
 395     BOOLEAN NpxPresent;
 396     ULONG FeatureBits;
 397     ULONG PageDirectory[2];
 398     PVOID DpcStack;
 399     ULONG Vendor[3];
 400
 401     /* Detect and set the CPU Type */
 402     KiSetProcessorType();
 403
 404     /* Set CR0 features based on detected CPU */
 405     KiSetCR0Bits();
 406
 407     /* Check if an FPU is present */
 408     NpxPresent = KiIsNpxPresent();
 409
 410     /* Initialize the Power Management Support for this PRCB */
 411     PoInitializePrcb(Prcb);
 412
 413     /* Bugcheck if this is a 386 CPU */
 414     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 415
 416     /* Get the processor features for the CPU */
 417     FeatureBits = KiGetFeatureBits();
 418
 419     /* Set the default NX policy (opt-in) */
 420     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 421
 422     /* Check if NPX is always on */
 423     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 424     {
 425         /* Set it always on */
 426         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 427         FeatureBits |= KF_NX_ENABLED;
 428     }
 429     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 430     {
 431         /* Set it in opt-out mode */
 432         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 433         FeatureBits |= KF_NX_ENABLED;
 434     }
 435     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 436              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 437     {
 438         /* Set the feature bits */
 439         FeatureBits |= KF_NX_ENABLED;
 440     }
 441     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 442              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 443     {
 444         /* Set disabled mode */
 445         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 446         FeatureBits |= KF_NX_DISABLED;
 447     }
 448
 449     /* Save feature bits */
 450     Prcb->FeatureBits = FeatureBits;
 451
 452     /* Save CPU state */
 453     KiSaveProcessorControlState(&Prcb->ProcessorState);
 454
 455     /* Get cache line information for this CPU */
 456     KiGetCacheInformation();
 457
 458     /* Initialize spinlocks and DPC data */
 459     KiInitSpinLocks(Prcb, Number);
 460
 461     /* Check if this is the Boot CPU */
 462     if (!Number)
 463     {
 464         /* Set Node Data */
 465         KeNodeBlock[0] = &KiNode0;
 466         Prcb->ParentNode = KeNodeBlock[0];
 467         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 468
 469         /* Set boot-level flags */
 470         KeI386NpxPresent = NpxPresent;
 471         KeI386CpuType = Prcb->CpuType;
 472         KeI386CpuStep = Prcb->CpuStep;
 473         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 474         KeProcessorLevel = (USHORT)Prcb->CpuType;
 475         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 476         KeFeatureBits = FeatureBits;
 477         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 478         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 479
 480         /* Detect 8-byte compare exchange support */
 481         if (!(KeFeatureBits & KF_CMPXCHG8B))
 482         {
 483             /* Copy the vendor string */
 484             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 485
 486             /* Bugcheck the system. Windows *requires* this */
 487             KeBugCheckEx(0x5D,
 488                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 489                          Vendor[0],
 490                          Vendor[1],
 491                          Vendor[2]);
 492         }
 493
 494         /* Set the current MP Master KPRCB to the Boot PRCB */
 495         Prcb->MultiThreadSetMaster = Prcb;
 496
 497         /* Lower to APC_LEVEL */
 498         KeLowerIrql(APC_LEVEL);
 499
 500         /* Initialize some spinlocks */
 501         KeInitializeSpinLock(&KiFreezeExecutionLock);
 502         KeInitializeSpinLock(&Ki486CompatibilityLock);
 503
 504         /* Initialize portable parts of the OS */
 505         KiInitSystem();
 506
 507         /* Initialize the Idle Process and the Process Listhead */
 508         InitializeListHead(&KiProcessListHead);
 509         PageDirectory[0] = 0;
 510         PageDirectory[1] = 0;
 511         KeInitializeProcess(InitProcess,
 512                             0,
 513                             0xFFFFFFFF,
 514                             PageDirectory,
 515                             FALSE);
 516         InitProcess->QuantumReset = MAXCHAR;
 517     }
 518     else
 519     {
 520         /* FIXME */
 521         DPRINT1("SMP Boot support not yet present\n");
 522     }
 523
 524     /* Setup the Idle Thread */
 525     KeInitializeThread(InitProcess,
 526                        InitThread,
 527                        NULL,
 528                        NULL,
 529                        NULL,
 530                        NULL,
 531                        NULL,
 532                        IdleStack);
 533     InitThread->NextProcessor = Number;
 534     InitThread->Priority = HIGH_PRIORITY;
 535     InitThread->State = Running;
 536     InitThread->Affinity = 1 << Number;
 537     InitThread->WaitIrql = DISPATCH_LEVEL;
 538     InitProcess->ActiveProcessors = 1 << Number;
 539
 540     /* HACK for MmUpdatePageDir */
 541     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 542
 543     /* Set basic CPU Features that user mode can read */
 544     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 545         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 546     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 547         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 548     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 549         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 550     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 551         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 552     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 553         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 554     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 555         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 556
 557     /* Set up the thread-related fields in the PRCB */
 558     Prcb->CurrentThread = InitThread;
 559     Prcb->NextThread = NULL;
 560     Prcb->IdleThread = InitThread;
 561
 562     /* Initialize the Kernel Executive */
 563     ExpInitializeExecutive(Number, LoaderBlock);
 564
 565     /* Only do this on the boot CPU */
 566     if (!Number)
 567     {
 568         /* Calculate the time reciprocal */
 569         KiTimeIncrementReciprocal =
 570             KiComputeReciprocal(KeMaximumIncrement,
 571                                 &KiTimeIncrementShiftCount);
 572
 573         /* Update DPC Values in case they got updated by the executive */
 574         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 575         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 576         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 577
 578         /* Allocate the DPC Stack */
 579         DpcStack = MmCreateKernelStack(FALSE, 0);
 580         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 581         Prcb->DpcStack = DpcStack;
 582
 583         /* Allocate the IOPM save area. */
 584         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 585                                                   PAGE_SIZE * 2,
 586                                                   '  eK');
 587         if (!Ki386IopmSaveArea)
 588         {
 589             /* Bugcheck. We need this for V86/VDM support. */
 590             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 591         }
 592     }
 593
 594     /* Raise to Dispatch */
 595     KfRaiseIrql(DISPATCH_LEVEL);
 596
 597     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 598     KeSetPriorityThread(InitThread, 0);
 599
 600     /* If there's no thread scheduled, put this CPU in the Idle summary */
 601     KiAcquirePrcbLock(Prcb);
 602     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 603     KiReleasePrcbLock(Prcb);
 604
 605     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 606     KfRaiseIrql(HIGH_LEVEL);
 607     LoaderBlock->Prcb = 0;
 608 }
 609
 610 VOID
 611 FASTCALL
 612 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 613                          IN PKIDTENTRY *Idt,
 614                          IN PKIPCR *Pcr,
 615                          IN PKTSS *Tss)
 616 {
 617     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 618     KGDTENTRY TssSelector, PcrSelector;
 619     USHORT Tr, Fs;
 620
 621     /* Get GDT and IDT descriptors */
 622     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 623     __sidt(&IdtDescriptor.Limit);
 624
 625     /* Save IDT and GDT */
 626     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 627     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 628
 629     /* Get TSS and FS Selectors */
 630     Tr = Ke386GetTr();
 631     Fs = Ke386GetFs();
 632
 633     /* Get PCR Selector, mask it and get its GDT Entry */
 634     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 635
 636     /* Get the KPCR itself */
 637     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 638                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 639                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 640
 641     /* Get TSS Selector, mask it and get its GDT Entry */
 642     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 643
 644     /* Get the KTSS itself */
 645     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 646                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 647                               TssSelector.HighWord.Bytes.BaseHi << 24);
 648 }
 649
 650 VOID
 651 NTAPI
 652 KiSystemStartupBootStack(VOID)
 653 {
 654     PKTHREAD Thread;
 655
 656     /* Initialize the kernel for the current CPU */
 657     KiInitializeKernel(&KiInitialProcess.Pcb,
 658                        (PKTHREAD)KeLoaderBlock->Thread,
 659                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 660                        (PKPRCB)__readfsdword(KPCR_PRCB),
 661                        KeNumberProcessors - 1,
 662                        KeLoaderBlock);
 663
 664     /* Set the priority of this thread to 0 */
 665     Thread = KeGetCurrentThread();
 666     Thread->Priority = 0;
 667
 668     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 669     _enable();
 670     KfLowerIrql(DISPATCH_LEVEL);
 671
 672     /* Set the right wait IRQL */
 673     Thread->WaitIrql = DISPATCH_LEVEL;
 674
 675     /* Jump into the idle loop */
 676     KiIdleLoop();
 677 }
 678
 679 VOID
 680 NTAPI
 681 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 682 {
 683     ULONG Cpu;
 684     PKTHREAD InitialThread;
 685     ULONG InitialStack;
 686     PKGDTENTRY Gdt;
 687     PKIDTENTRY Idt;
 688     KIDTENTRY NmiEntry, DoubleFaultEntry;
 689     PKTSS Tss;
 690     PKIPCR Pcr;
 691
 692     /* Check if we are being booted from FreeLDR */
 693     if (!((ULONG_PTR)LoaderBlock & 0x80000000)) KiRosPrepareForSystemStartup((PROS_LOADER_PARAMETER_BLOCK)LoaderBlock);
 694
 695     /* Save the loader block and get the current CPU */
 696     KeLoaderBlock = LoaderBlock;
 697     Cpu = KeNumberProcessors;
 698     if (!Cpu)
 699     {
 700         /* If this is the boot CPU, set FS and the CPU Number*/
 701         Ke386SetFs(KGDT_R0_PCR);
 702         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 703
 704         /* Set the initial stack and idle thread as well */
 705         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 706         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 707     }
 708
 709     /* Save the initial thread and stack */
 710     InitialStack = LoaderBlock->KernelStack;
 711     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 712
 713     /* Clean the APC List Head */
 714     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 715
 716     /* Initialize the machine type */
 717     KiInitializeMachineType();
 718
 719     /* Skip initial setup if this isn't the Boot CPU */
 720     if (Cpu) goto AppCpuInit;
 721
 722     /* Get GDT, IDT, PCR and TSS pointers */
 723     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 724
 725     /* Setup the TSS descriptors and entries */
 726     Ki386InitializeTss(Tss, Idt, Gdt);
 727
 728     /* Initialize the PCR */
 729     RtlZeroMemory(Pcr, PAGE_SIZE);
 730     KiInitializePcr(Cpu,
 731                     Pcr,
 732                     Idt,
 733                     Gdt,
 734                     Tss,
 735                     InitialThread,
 736                     (PVOID)KiDoubleFaultStack);
 737
 738     /* Set us as the current process */
 739     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 740
 741     /* Clear DR6/7 to cleanup bootloader debugging */
 742     __writefsdword(KPCR_TEB, 0);
 743     __writefsdword(KPCR_DR6, 0);
 744     __writefsdword(KPCR_DR7, 0);
 745
 746     /* Setup the IDT */
 747     KeInitExceptions();
 748
 749     /* Load Ring 3 selectors for DS/ES */
 750     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 751     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 752
 753     /* Save NMI and double fault traps */
 754     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 755     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 756
 757     /* Copy kernel's trap handlers */
 758     RtlCopyMemory(Idt,
 759                   (PVOID)KiIdtDescriptor.Base,
 760                   KiIdtDescriptor.Limit + 1);
 761
 762     /* Restore NMI and double fault */
 763     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 764     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 765
 766 AppCpuInit:
 767     /* Loop until we can release the freeze lock */
 768     do
 769     {
 770         /* Loop until execution can continue */
 771         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 772     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 773
 774     /* Setup CPU-related fields */
 775     __writefsdword(KPCR_NUMBER, Cpu);
 776     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 777     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 778     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 779
 780     /* Initialize the Processor with HAL */
 781     HalInitializeProcessor(Cpu, KeLoaderBlock);
 782
 783     /* Set active processors */
 784     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 785     KeNumberProcessors++;
 786
 787     /* Check if this is the boot CPU */
 788     if (!Cpu)
 789     {
 790         /* Initialize debugging system */
 791         KdInitSystem(0, KeLoaderBlock);
 792
 793         /* Check for break-in */
 794         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 795     }
 796
 797     /* Raise to HIGH_LEVEL */
 798     KfRaiseIrql(HIGH_LEVEL);
 799
 800     /* Switch to new kernel stack and start kernel bootstrapping */
 801     KiSwitchToBootStack(InitialStack & ~3);
 802 }