ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR P0BootStackData[KERNEL_STACK_SIZE] __attribute__((aligned (16))) = {0};
  20 UCHAR KiDoubleFaultStackData[KERNEL_STACK_SIZE] __attribute__((aligned (16))) = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 VOID
  35 NTAPI
  36 KiInitMachineDependent(VOID)
  37 {
  38     ULONG CpuCount;
  39     BOOLEAN FbCaching = FALSE;
  40     NTSTATUS Status;
  41     ULONG ReturnLength;
  42     ULONG i, Affinity, Sample = 0;
  43     PFX_SAVE_AREA FxSaveArea;
  44     ULONG MXCsrMask = 0xFFBF;
  45     ULONG Dummy;
  46     KI_SAMPLE_MAP Samples[4];
  47     PKI_SAMPLE_MAP CurrentSample = Samples;
  48
  49     /* Check for large page support */
  50     if (KeFeatureBits & KF_LARGE_PAGE)
  51     {
  52         /* FIXME: Support this */
  53         DPRINT1("Large Page support detected but not yet taken advantage of\n");
  54     }
  55
  56     /* Check for global page support */
  57     if (KeFeatureBits & KF_GLOBAL_PAGE)
  58     {
  59         /* Do an IPI to enable it on all CPUs */
  60         CpuCount = KeNumberProcessors;
  61         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  62     }
  63
  64     /* Check for PAT and/or MTRR support */
  65     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  66     {
  67         /* Query the HAL to make sure we can use it */
  68         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  69                                            sizeof(BOOLEAN),
  70                                            &FbCaching,
  71                                            &ReturnLength);
  72         if ((NT_SUCCESS(Status)) && (FbCaching))
  73         {
  74             /* We can't, disable it */
  75             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  76         }
  77     }
  78
  79     /* Check for PAT support and enable it */
  80     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  81
  82     /* Assume no errata for now */
  83     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  84
  85     /* Check if we have an NPX */
  86     if (KeI386NpxPresent)
  87     {
  88         /* Loop every CPU */
  89         i = KeActiveProcessors;
  90         for (Affinity = 1; i; Affinity <<= 1)
  91         {
  92             /* Check if this is part of the set */
  93             if (i & Affinity)
  94             {
  95                 /* Run on this CPU */
  96                 i &= ~Affinity;
  97                 KeSetSystemAffinityThread(Affinity);
  98
  99                 /* Detect FPU errata */
 100                 if (KiIsNpxErrataPresent())
 101                 {
 102                     /* Disable NPX support */
 103                     KeI386NpxPresent = FALSE;
 104                     SharedUserData->
 105                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 106                         TRUE;
 107                     break;
 108                 }
 109             }
 110         }
 111     }
 112
 113     /* If there's no NPX, then we're emulating the FPU */
 114     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 115         !KeI386NpxPresent;
 116
 117     /* Check if there's no NPX, so that we can disable associated features */
 118     if (!KeI386NpxPresent)
 119     {
 120         /* Remove NPX-related bits */
 121         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 122
 123         /* Disable kernel flags */
 124         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 125
 126         /* Disable processor features that might've been set until now */
 127         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 128         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 129         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 130         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 131         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 132     }
 133
 134     /* Check for CR4 support */
 135     if (KeFeatureBits & KF_CR4)
 136     {
 137         /* Do an IPI call to enable the Debug Exceptions */
 138         CpuCount = KeNumberProcessors;
 139         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 140     }
 141
 142     /* Check if FXSR was found */
 143     if (KeFeatureBits & KF_FXSR)
 144     {
 145         /* Do an IPI call to enable the FXSR */
 146         CpuCount = KeNumberProcessors;
 147         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 148
 149         /* Check if XMM was found too */
 150         if (KeFeatureBits & KF_XMMI)
 151         {
 152             /* Do an IPI call to enable XMMI exceptions */
 153             CpuCount = KeNumberProcessors;
 154             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 155
 156             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 157
 158             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 159             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 160         }
 161     }
 162
 163     /* Check for, and enable SYSENTER support */
 164     KiRestoreFastSyscallReturnState();
 165
 166     /* Loop every CPU */
 167     i = KeActiveProcessors;
 168     for (Affinity = 1; i; Affinity <<= 1)
 169     {
 170         /* Check if this is part of the set */
 171         if (i & Affinity)
 172         {
 173             /* Run on this CPU */
 174             i &= ~Affinity;
 175             KeSetSystemAffinityThread(Affinity);
 176
 177             /* Reset MHz to 0 for this CPU */
 178             KeGetCurrentPrcb()->MHz = 0;
 179
 180             /* Check if we can use RDTSC */
 181             if (KeFeatureBits & KF_RDTSC)
 182             {
 183                 /* Start sampling loop */
 184                 for (;;)
 185                 {
 186                     /* Do a dummy CPUID to start the sample */
 187                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 188
 189                     /* Fill out the starting data */
 190                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 191                     CurrentSample->TSCStart = __rdtsc();
 192                     CurrentSample->PerfFreq.QuadPart = -50000;
 193
 194                     /* Sleep for this sample */
 195                     KeDelayExecutionThread(KernelMode,
 196                                            FALSE,
 197                                            &CurrentSample->PerfFreq);
 198
 199                     /* Do another dummy CPUID */
 200                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 201
 202                     /* Fill out the ending data */
 203                     CurrentSample->PerfEnd =
 204                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 205                     CurrentSample->TSCEnd = __rdtsc();
 206
 207                     /* Calculate the differences */
 208                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 209                                                CurrentSample->PerfStart.QuadPart;
 210                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 211                                               CurrentSample->TSCStart;
 212
 213                     /* Compute CPU Speed */
 214                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 215                                                   CurrentSample->
 216                                                   PerfFreq.QuadPart + 500000) /
 217                                                  (CurrentSample->PerfDelta *
 218                                                   1000000));
 219
 220                     /* Check if this isn't the first sample */
 221                     if (Sample)
 222                     {
 223                         /* Check if we got a good precision within 1MHz */
 224                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 225                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 226                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 227                         {
 228                             /* We did, stop sampling */
 229                             break;
 230                         }
 231                     }
 232
 233                     /* Move on */
 234                     CurrentSample++;
 235                     Sample++;
 236
 237                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 238                     {
 239                         /* Restart */
 240                         CurrentSample = Samples;
 241                         Sample = 0;
 242                     }
 243                 }
 244
 245                 /* Save the CPU Speed */
 246                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 247             }
 248
 249             /* Check if we have MTRR */
 250             if (KeFeatureBits & KF_MTRR)
 251             {
 252                 /* Then manually initialize MTRR for the CPU */
 253                 KiInitializeMTRR(i ? FALSE : TRUE);
 254             }
 255
 256             /* Check if we have AMD MTRR and initialize it for the CPU */
 257             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 258
 259             /* Check if this is a buggy Pentium and apply the fixup if so */
 260             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 261
 262             /* Check if the CPU supports FXSR */
 263             if (KeFeatureBits & KF_FXSR)
 264             {
 265                 /* Get the current thread NPX state */
 266                 FxSaveArea = (PVOID)
 267                              ((ULONG_PTR)KeGetCurrentThread()->InitialStack -
 268                              NPX_FRAME_LENGTH);
 269
 270                 /* Clear initial MXCsr mask */
 271                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 272
 273                 /* Save the current NPX State */
 274 #ifdef __GNUC__
 275                 asm volatile("fxsave %0\n\t" : "=m" (*FxSaveArea));
 276 #else
 277                 __asm fxsave [FxSaveArea]
 278 #endif
 279                 /* Check if the current mask doesn't match the reserved bits */
 280                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 281                 {
 282                     /* Then use whatever it's holding */
 283                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 284                 }
 285
 286                 /* Check if nobody set the kernel-wide mask */
 287                 if (!KiMXCsrMask)
 288                 {
 289                     /* Then use the one we calculated above */
 290                     KiMXCsrMask = MXCsrMask;
 291                 }
 292                 else
 293                 {
 294                     /* Was it set to the same value we found now? */
 295                     if (KiMXCsrMask != MXCsrMask)
 296                     {
 297                         /* No, something is definitely wrong */
 298                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 299                                      KF_FXSR,
 300                                      KiMXCsrMask,
 301                                      MXCsrMask,
 302                                      0);
 303                     }
 304                 }
 305
 306                 /* Now set the kernel mask */
 307                 KiMXCsrMask &= MXCsrMask;
 308             }
 309         }
 310     }
 311
 312     /* Return affinity back to where it was */
 313     KeRevertToUserAffinityThread();
 314
 315     /* NT allows limiting the duration of an ISR with a registry key */
 316     if (KiTimeLimitIsrMicroseconds)
 317     {
 318         /* FIXME: TODO */
 319         DPRINT1("ISR Time Limit not yet supported\n");
 320     }
 321
 322     /* Set CR0 features based on detected CPU */
 323     KiSetCR0Bits();
 324 }
 325
 326 VOID
 327 NTAPI
 328 KiInitializePcr(IN ULONG ProcessorNumber,
 329                 IN PKIPCR Pcr,
 330                 IN PKIDTENTRY Idt,
 331                 IN PKGDTENTRY Gdt,
 332                 IN PKTSS Tss,
 333                 IN PKTHREAD IdleThread,
 334                 IN PVOID DpcStack)
 335 {
 336     /* Setup the TIB */
 337     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 338     Pcr->NtTib.StackBase = 0;
 339     Pcr->NtTib.StackLimit = 0;
 340     Pcr->NtTib.Self = NULL;
 341
 342     /* Set the Current Thread */
 343     Pcr->PrcbData.CurrentThread = IdleThread;
 344
 345     /* Set pointers to ourselves */
 346     Pcr->Self = (PKPCR)Pcr;
 347     Pcr->Prcb = &Pcr->PrcbData;
 348
 349     /* Set the PCR Version */
 350     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 351     Pcr->MinorVersion = PCR_MINOR_VERSION;
 352
 353     /* Set the PCRB Version */
 354     Pcr->PrcbData.MajorVersion = 1;
 355     Pcr->PrcbData.MinorVersion = 1;
 356
 357     /* Set the Build Type */
 358     Pcr->PrcbData.BuildType = 0;
 359 #ifndef CONFIG_SMP
 360     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 361 #endif
 362 #if DBG
 363     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 364 #endif
 365
 366     /* Set the Processor Number and current Processor Mask */
 367     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 368     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 369
 370     /* Set the PRCB for this Processor */
 371     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 372
 373     /* Start us out at PASSIVE_LEVEL */
 374     Pcr->Irql = PASSIVE_LEVEL;
 375
 376     /* Set the GDI, IDT, TSS and DPC Stack */
 377     Pcr->GDT = (PVOID)Gdt;
 378     Pcr->IDT = Idt;
 379     Pcr->TSS = Tss;
 380     Pcr->TssCopy = Tss;
 381     Pcr->PrcbData.DpcStack = DpcStack;
 382
 383     /* Setup the processor set */
 384     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 385 }
 386
 387 VOID
 388 NTAPI
 389 KiInitializeKernel(IN PKPROCESS InitProcess,
 390                    IN PKTHREAD InitThread,
 391                    IN PVOID IdleStack,
 392                    IN PKPRCB Prcb,
 393                    IN CCHAR Number,
 394                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 395 {
 396     BOOLEAN NpxPresent;
 397     ULONG FeatureBits;
 398     ULONG PageDirectory[2];
 399     PVOID DpcStack;
 400     ULONG Vendor[3];
 401
 402     /* Detect and set the CPU Type */
 403     KiSetProcessorType();
 404
 405     /* Check if an FPU is present */
 406     NpxPresent = KiIsNpxPresent();
 407
 408     /* Initialize the Power Management Support for this PRCB */
 409     PoInitializePrcb(Prcb);
 410
 411     /* Bugcheck if this is a 386 CPU */
 412     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 413
 414     /* Get the processor features for the CPU */
 415     FeatureBits = KiGetFeatureBits();
 416
 417     /* Set the default NX policy (opt-in) */
 418     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 419
 420     /* Check if NPX is always on */
 421     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 422     {
 423         /* Set it always on */
 424         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 425         FeatureBits |= KF_NX_ENABLED;
 426     }
 427     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 428     {
 429         /* Set it in opt-out mode */
 430         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 431         FeatureBits |= KF_NX_ENABLED;
 432     }
 433     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 434              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 435     {
 436         /* Set the feature bits */
 437         FeatureBits |= KF_NX_ENABLED;
 438     }
 439     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 440              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 441     {
 442         /* Set disabled mode */
 443         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 444         FeatureBits |= KF_NX_DISABLED;
 445     }
 446
 447     /* Save feature bits */
 448     Prcb->FeatureBits = FeatureBits;
 449
 450     /* Save CPU state */
 451     KiSaveProcessorControlState(&Prcb->ProcessorState);
 452
 453     /* Get cache line information for this CPU */
 454     KiGetCacheInformation();
 455
 456     /* Initialize spinlocks and DPC data */
 457     KiInitSpinLocks(Prcb, Number);
 458
 459     /* Check if this is the Boot CPU */
 460     if (!Number)
 461     {
 462         /* Set Node Data */
 463         KeNodeBlock[0] = &KiNode0;
 464         Prcb->ParentNode = KeNodeBlock[0];
 465         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 466
 467         /* Set boot-level flags */
 468         KeI386NpxPresent = NpxPresent;
 469         KeI386CpuType = Prcb->CpuType;
 470         KeI386CpuStep = Prcb->CpuStep;
 471         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 472         KeProcessorLevel = (USHORT)Prcb->CpuType;
 473         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 474         KeFeatureBits = FeatureBits;
 475         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 476         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 477
 478         /* Detect 8-byte compare exchange support */
 479         if (!(KeFeatureBits & KF_CMPXCHG8B))
 480         {
 481             /* Copy the vendor string */
 482             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 483
 484             /* Bugcheck the system. Windows *requires* this */
 485             KeBugCheckEx(0x5D,
 486                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 487                          Vendor[0],
 488                          Vendor[1],
 489                          Vendor[2]);
 490         }
 491
 492         /* Set the current MP Master KPRCB to the Boot PRCB */
 493         Prcb->MultiThreadSetMaster = Prcb;
 494
 495         /* Lower to APC_LEVEL */
 496         KeLowerIrql(APC_LEVEL);
 497
 498         /* Initialize some spinlocks */
 499         KeInitializeSpinLock(&KiFreezeExecutionLock);
 500         KeInitializeSpinLock(&Ki486CompatibilityLock);
 501
 502         /* Initialize portable parts of the OS */
 503         KiInitSystem();
 504
 505         /* Initialize the Idle Process and the Process Listhead */
 506         InitializeListHead(&KiProcessListHead);
 507         PageDirectory[0] = 0;
 508         PageDirectory[1] = 0;
 509         KeInitializeProcess(InitProcess,
 510                             0,
 511                             0xFFFFFFFF,
 512                             PageDirectory,
 513                             FALSE);
 514         InitProcess->QuantumReset = MAXCHAR;
 515     }
 516     else
 517     {
 518         /* FIXME */
 519         DPRINT1("SMP Boot support not yet present\n");
 520     }
 521
 522     /* Setup the Idle Thread */
 523     KeInitializeThread(InitProcess,
 524                        InitThread,
 525                        NULL,
 526                        NULL,
 527                        NULL,
 528                        NULL,
 529                        NULL,
 530                        IdleStack);
 531     InitThread->NextProcessor = Number;
 532     InitThread->Priority = HIGH_PRIORITY;
 533     InitThread->State = Running;
 534     InitThread->Affinity = 1 << Number;
 535     InitThread->WaitIrql = DISPATCH_LEVEL;
 536     InitProcess->ActiveProcessors = 1 << Number;
 537
 538     /* HACK for MmUpdatePageDir */
 539     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 540
 541     /* Set basic CPU Features that user mode can read */
 542     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 543         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 544     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 545         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 546     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 547         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 548     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 549         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 550     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 551         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 552     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 553         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 554
 555     /* Set up the thread-related fields in the PRCB */
 556     Prcb->CurrentThread = InitThread;
 557     Prcb->NextThread = NULL;
 558     Prcb->IdleThread = InitThread;
 559
 560     /* Initialize the Kernel Executive */
 561     ExpInitializeExecutive(Number, LoaderBlock);
 562
 563     /* Only do this on the boot CPU */
 564     if (!Number)
 565     {
 566         /* Calculate the time reciprocal */
 567         KiTimeIncrementReciprocal =
 568             KiComputeReciprocal(KeMaximumIncrement,
 569                                 &KiTimeIncrementShiftCount);
 570
 571         /* Update DPC Values in case they got updated by the executive */
 572         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 573         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 574         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 575
 576         /* Allocate the DPC Stack */
 577         DpcStack = MmCreateKernelStack(FALSE, 0);
 578         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 579         Prcb->DpcStack = DpcStack;
 580
 581         /* Allocate the IOPM save area. */
 582         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 583                                                   PAGE_SIZE * 2,
 584                                                   '  eK');
 585         if (!Ki386IopmSaveArea)
 586         {
 587             /* Bugcheck. We need this for V86/VDM support. */
 588             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 589         }
 590     }
 591
 592     /* Raise to Dispatch */
 593     KfRaiseIrql(DISPATCH_LEVEL);
 594
 595     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 596     KeSetPriorityThread(InitThread, 0);
 597
 598     /* If there's no thread scheduled, put this CPU in the Idle summary */
 599     KiAcquirePrcbLock(Prcb);
 600     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 601     KiReleasePrcbLock(Prcb);
 602
 603     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 604     KfRaiseIrql(HIGH_LEVEL);
 605     LoaderBlock->Prcb = 0;
 606 }
 607
 608 VOID
 609 FASTCALL
 610 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 611                          IN PKIDTENTRY *Idt,
 612                          IN PKIPCR *Pcr,
 613                          IN PKTSS *Tss)
 614 {
 615     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 616     KGDTENTRY TssSelector, PcrSelector;
 617     USHORT Tr, Fs;
 618
 619     /* Get GDT and IDT descriptors */
 620     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 621     __sidt(&IdtDescriptor.Limit);
 622
 623     /* Save IDT and GDT */
 624     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 625     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 626
 627     /* Get TSS and FS Selectors */
 628     Tr = Ke386GetTr();
 629     Fs = Ke386GetFs();
 630
 631     /* Get PCR Selector, mask it and get its GDT Entry */
 632     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 633
 634     /* Get the KPCR itself */
 635     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 636                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 637                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 638
 639     /* Get TSS Selector, mask it and get its GDT Entry */
 640     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 641
 642     /* Get the KTSS itself */
 643     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 644                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 645                               TssSelector.HighWord.Bytes.BaseHi << 24);
 646 }
 647
 648 VOID
 649 NTAPI
 650 KiSystemStartupBootStack(VOID)
 651 {
 652     PKTHREAD Thread;
 653
 654     /* Initialize the kernel for the current CPU */
 655     KiInitializeKernel(&KiInitialProcess.Pcb,
 656                        (PKTHREAD)KeLoaderBlock->Thread,
 657                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 658                        (PKPRCB)__readfsdword(KPCR_PRCB),
 659                        KeNumberProcessors - 1,
 660                        KeLoaderBlock);
 661
 662     /* Set the priority of this thread to 0 */
 663     Thread = KeGetCurrentThread();
 664     Thread->Priority = 0;
 665
 666     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 667     _enable();
 668     KfLowerIrql(DISPATCH_LEVEL);
 669
 670     /* Set the right wait IRQL */
 671     Thread->WaitIrql = DISPATCH_LEVEL;
 672
 673     /* Jump into the idle loop */
 674     KiIdleLoop();
 675 }
 676
 677 VOID
 678 NTAPI
 679 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 680 {
 681     ULONG Cpu;
 682     PKTHREAD InitialThread;
 683     ULONG InitialStack;
 684     PKGDTENTRY Gdt;
 685     PKIDTENTRY Idt;
 686     KIDTENTRY NmiEntry, DoubleFaultEntry;
 687     PKTSS Tss;
 688     PKIPCR Pcr;
 689
 690     /* Boot cycles timestamp */
 691     BootCycles = __rdtsc();
 692
 693     /* Check if we are being booted from FreeLDR */
 694     if (!((ULONG_PTR)LoaderBlock & 0x80000000)) KiRosPrepareForSystemStartup((PROS_LOADER_PARAMETER_BLOCK)LoaderBlock);
 695
 696     /* Save the loader block and get the current CPU */
 697     KeLoaderBlock = LoaderBlock;
 698     Cpu = KeNumberProcessors;
 699     if (!Cpu)
 700     {
 701         /* If this is the boot CPU, set FS and the CPU Number*/
 702         Ke386SetFs(KGDT_R0_PCR);
 703         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 704
 705         /* Set the initial stack and idle thread as well */
 706         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 707         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 708     }
 709
 710     /* Save the initial thread and stack */
 711     InitialStack = LoaderBlock->KernelStack;
 712     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 713
 714     /* Clean the APC List Head */
 715     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 716
 717     /* Initialize the machine type */
 718     KiInitializeMachineType();
 719
 720     /* Skip initial setup if this isn't the Boot CPU */
 721     if (Cpu) goto AppCpuInit;
 722
 723     /* Get GDT, IDT, PCR and TSS pointers */
 724     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 725
 726     /* Setup the TSS descriptors and entries */
 727     Ki386InitializeTss(Tss, Idt, Gdt);
 728
 729     /* Initialize the PCR */
 730     RtlZeroMemory(Pcr, PAGE_SIZE);
 731     KiInitializePcr(Cpu,
 732                     Pcr,
 733                     Idt,
 734                     Gdt,
 735                     Tss,
 736                     InitialThread,
 737                     (PVOID)KiDoubleFaultStack);
 738
 739     /* Set us as the current process */
 740     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 741
 742     /* Clear DR6/7 to cleanup bootloader debugging */
 743     __writefsdword(KPCR_TEB, 0);
 744     __writefsdword(KPCR_DR6, 0);
 745     __writefsdword(KPCR_DR7, 0);
 746
 747     /* Setup the IDT */
 748     KeInitExceptions();
 749
 750     /* Load Ring 3 selectors for DS/ES */
 751     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 752     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 753
 754     /* Save NMI and double fault traps */
 755     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 756     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 757
 758     /* Copy kernel's trap handlers */
 759     RtlCopyMemory(Idt,
 760                   (PVOID)KiIdtDescriptor.Base,
 761                   KiIdtDescriptor.Limit + 1);
 762
 763     /* Restore NMI and double fault */
 764     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 765     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 766
 767 AppCpuInit:
 768     /* Loop until we can release the freeze lock */
 769     do
 770     {
 771         /* Loop until execution can continue */
 772         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 773     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 774
 775     /* Setup CPU-related fields */
 776     __writefsdword(KPCR_NUMBER, Cpu);
 777     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 778     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 779     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 780
 781     /* Initialize the Processor with HAL */
 782     HalInitializeProcessor(Cpu, KeLoaderBlock);
 783
 784     /* Set active processors */
 785     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 786     KeNumberProcessors++;
 787
 788     /* Check if this is the boot CPU */
 789     if (!Cpu)
 790     {
 791         /* Initialize debugging system */
 792         KdInitSystem(0, KeLoaderBlock);
 793
 794         /* Check for break-in */
 795         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 796     }
 797
 798     /* Raise to HIGH_LEVEL */
 799     KfRaiseIrql(HIGH_LEVEL);
 800
 801     /* Switch to new kernel stack and start kernel bootstrapping */
 802     KiSwitchToBootStack(InitialStack & ~3);
 803 }