reactos/ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  20 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 VOID
  35 NTAPI
  36 INIT_FUNCTION
  37 KiInitMachineDependent(VOID)
  38 {
  39     ULONG CpuCount;
  40     BOOLEAN FbCaching = FALSE;
  41     NTSTATUS Status;
  42     ULONG ReturnLength;
  43     ULONG i, Affinity, Sample = 0;
  44     PFX_SAVE_AREA FxSaveArea;
  45     ULONG MXCsrMask = 0xFFBF;
  46     ULONG Dummy;
  47     KI_SAMPLE_MAP Samples[4];
  48     PKI_SAMPLE_MAP CurrentSample = Samples;
  49
  50     /* Check for large page support */
  51     if (KeFeatureBits & KF_LARGE_PAGE)
  52     {
  53         /* FIXME: Support this */
  54         DPRINT1("Large Page support detected but not yet taken advantage of\n");
  55     }
  56
  57     /* Check for global page support */
  58     if (KeFeatureBits & KF_GLOBAL_PAGE)
  59     {
  60         /* Do an IPI to enable it on all CPUs */
  61         CpuCount = KeNumberProcessors;
  62         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  63     }
  64
  65     /* Check for PAT and/or MTRR support */
  66     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  67     {
  68         /* Query the HAL to make sure we can use it */
  69         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  70                                            sizeof(BOOLEAN),
  71                                            &FbCaching,
  72                                            &ReturnLength);
  73         if ((NT_SUCCESS(Status)) && (FbCaching))
  74         {
  75             /* We can't, disable it */
  76             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  77         }
  78     }
  79
  80     /* Check for PAT support and enable it */
  81     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  82
  83     /* Assume no errata for now */
  84     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  85
  86     /* Check if we have an NPX */
  87     if (KeI386NpxPresent)
  88     {
  89         /* Loop every CPU */
  90         i = KeActiveProcessors;
  91         for (Affinity = 1; i; Affinity <<= 1)
  92         {
  93             /* Check if this is part of the set */
  94             if (i & Affinity)
  95             {
  96                 /* Run on this CPU */
  97                 i &= ~Affinity;
  98                 KeSetSystemAffinityThread(Affinity);
  99
 100                 /* Detect FPU errata */
 101                 if (KiIsNpxErrataPresent())
 102                 {
 103                     /* Disable NPX support */
 104                     KeI386NpxPresent = FALSE;
 105                     SharedUserData->
 106                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 107                         TRUE;
 108                     break;
 109                 }
 110             }
 111         }
 112     }
 113
 114     /* If there's no NPX, then we're emulating the FPU */
 115     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 116         !KeI386NpxPresent;
 117
 118     /* Check if there's no NPX, so that we can disable associated features */
 119     if (!KeI386NpxPresent)
 120     {
 121         /* Remove NPX-related bits */
 122         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 123
 124         /* Disable kernel flags */
 125         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 126
 127         /* Disable processor features that might've been set until now */
 128         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 129         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 130         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 131         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 132         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 133     }
 134
 135     /* Check for CR4 support */
 136     if (KeFeatureBits & KF_CR4)
 137     {
 138         /* Do an IPI call to enable the Debug Exceptions */
 139         CpuCount = KeNumberProcessors;
 140         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 141     }
 142
 143     /* Check if FXSR was found */
 144     if (KeFeatureBits & KF_FXSR)
 145     {
 146         /* Do an IPI call to enable the FXSR */
 147         CpuCount = KeNumberProcessors;
 148         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 149
 150         /* Check if XMM was found too */
 151         if (KeFeatureBits & KF_XMMI)
 152         {
 153             /* Do an IPI call to enable XMMI exceptions */
 154             CpuCount = KeNumberProcessors;
 155             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 156
 157             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 158
 159             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 160             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 161         }
 162     }
 163
 164     /* Check for, and enable SYSENTER support */
 165     KiRestoreFastSyscallReturnState();
 166
 167     /* Loop every CPU */
 168     i = KeActiveProcessors;
 169     for (Affinity = 1; i; Affinity <<= 1)
 170     {
 171         /* Check if this is part of the set */
 172         if (i & Affinity)
 173         {
 174             /* Run on this CPU */
 175             i &= ~Affinity;
 176             KeSetSystemAffinityThread(Affinity);
 177
 178             /* Reset MHz to 0 for this CPU */
 179             KeGetCurrentPrcb()->MHz = 0;
 180
 181             /* Check if we can use RDTSC */
 182             if (KeFeatureBits & KF_RDTSC)
 183             {
 184                 /* Start sampling loop */
 185                 for (;;)
 186                 {
 187                     /* Do a dummy CPUID to start the sample */
 188                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 189
 190                     /* Fill out the starting data */
 191                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 192                     CurrentSample->TSCStart = __rdtsc();
 193                     CurrentSample->PerfFreq.QuadPart = -50000;
 194
 195                     /* Sleep for this sample */
 196                     KeDelayExecutionThread(KernelMode,
 197                                            FALSE,
 198                                            &CurrentSample->PerfFreq);
 199
 200                     /* Do another dummy CPUID */
 201                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 202
 203                     /* Fill out the ending data */
 204                     CurrentSample->PerfEnd =
 205                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 206                     CurrentSample->TSCEnd = __rdtsc();
 207
 208                     /* Calculate the differences */
 209                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 210                                                CurrentSample->PerfStart.QuadPart;
 211                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 212                                               CurrentSample->TSCStart;
 213
 214                     /* Compute CPU Speed */
 215                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 216                                                   CurrentSample->
 217                                                   PerfFreq.QuadPart + 500000) /
 218                                                  (CurrentSample->PerfDelta *
 219                                                   1000000));
 220
 221                     /* Check if this isn't the first sample */
 222                     if (Sample)
 223                     {
 224                         /* Check if we got a good precision within 1MHz */
 225                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 226                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 227                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 228                         {
 229                             /* We did, stop sampling */
 230                             break;
 231                         }
 232                     }
 233
 234                     /* Move on */
 235                     CurrentSample++;
 236                     Sample++;
 237
 238                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 239                     {
 240                         /* Restart */
 241                         CurrentSample = Samples;
 242                         Sample = 0;
 243                     }
 244                 }
 245
 246                 /* Save the CPU Speed */
 247                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 248             }
 249
 250             /* Check if we have MTRR */
 251             if (KeFeatureBits & KF_MTRR)
 252             {
 253                 /* Then manually initialize MTRR for the CPU */
 254                 KiInitializeMTRR(i ? FALSE : TRUE);
 255             }
 256
 257             /* Check if we have AMD MTRR and initialize it for the CPU */
 258             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 259
 260             /* Check if this is a buggy Pentium and apply the fixup if so */
 261             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 262
 263             /* Check if the CPU supports FXSR */
 264             if (KeFeatureBits & KF_FXSR)
 265             {
 266                 /* Get the current thread NPX state */
 267                 FxSaveArea = (PVOID)
 268                              ((ULONG_PTR)KeGetCurrentThread()->InitialStack -
 269                              NPX_FRAME_LENGTH);
 270
 271                 /* Clear initial MXCsr mask */
 272                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 273
 274                 /* Save the current NPX State */
 275 #ifdef __GNUC__
 276                 asm volatile("fxsave %0\n\t" : "=m" (*FxSaveArea));
 277 #else
 278                 __asm fxsave [FxSaveArea]
 279 #endif
 280                 /* Check if the current mask doesn't match the reserved bits */
 281                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 282                 {
 283                     /* Then use whatever it's holding */
 284                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 285                 }
 286
 287                 /* Check if nobody set the kernel-wide mask */
 288                 if (!KiMXCsrMask)
 289                 {
 290                     /* Then use the one we calculated above */
 291                     KiMXCsrMask = MXCsrMask;
 292                 }
 293                 else
 294                 {
 295                     /* Was it set to the same value we found now? */
 296                     if (KiMXCsrMask != MXCsrMask)
 297                     {
 298                         /* No, something is definitely wrong */
 299                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 300                                      KF_FXSR,
 301                                      KiMXCsrMask,
 302                                      MXCsrMask,
 303                                      0);
 304                     }
 305                 }
 306
 307                 /* Now set the kernel mask */
 308                 KiMXCsrMask &= MXCsrMask;
 309             }
 310         }
 311     }
 312
 313     /* Return affinity back to where it was */
 314     KeRevertToUserAffinityThread();
 315
 316     /* NT allows limiting the duration of an ISR with a registry key */
 317     if (KiTimeLimitIsrMicroseconds)
 318     {
 319         /* FIXME: TODO */
 320         DPRINT1("ISR Time Limit not yet supported\n");
 321     }
 322
 323     /* Set CR0 features based on detected CPU */
 324     KiSetCR0Bits();
 325 }
 326
 327 VOID
 328 NTAPI
 329 INIT_FUNCTION
 330 KiInitializePcr(IN ULONG ProcessorNumber,
 331                 IN PKIPCR Pcr,
 332                 IN PKIDTENTRY Idt,
 333                 IN PKGDTENTRY Gdt,
 334                 IN PKTSS Tss,
 335                 IN PKTHREAD IdleThread,
 336                 IN PVOID DpcStack)
 337 {
 338     /* Setup the TIB */
 339     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 340     Pcr->NtTib.StackBase = 0;
 341     Pcr->NtTib.StackLimit = 0;
 342     Pcr->NtTib.Self = NULL;
 343
 344     /* Set the Current Thread */
 345     Pcr->PrcbData.CurrentThread = IdleThread;
 346
 347     /* Set pointers to ourselves */
 348     Pcr->Self = (PKPCR)Pcr;
 349     Pcr->Prcb = &Pcr->PrcbData;
 350
 351     /* Set the PCR Version */
 352     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 353     Pcr->MinorVersion = PCR_MINOR_VERSION;
 354
 355     /* Set the PCRB Version */
 356     Pcr->PrcbData.MajorVersion = 1;
 357     Pcr->PrcbData.MinorVersion = 1;
 358
 359     /* Set the Build Type */
 360     Pcr->PrcbData.BuildType = 0;
 361 #ifndef CONFIG_SMP
 362     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 363 #endif
 364 #if DBG
 365     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 366 #endif
 367
 368     /* Set the Processor Number and current Processor Mask */
 369     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 370     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 371
 372     /* Set the PRCB for this Processor */
 373     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 374
 375     /* Start us out at PASSIVE_LEVEL */
 376     Pcr->Irql = PASSIVE_LEVEL;
 377
 378     /* Set the GDI, IDT, TSS and DPC Stack */
 379     Pcr->GDT = (PVOID)Gdt;
 380     Pcr->IDT = Idt;
 381     Pcr->TSS = Tss;
 382     Pcr->TssCopy = Tss;
 383     Pcr->PrcbData.DpcStack = DpcStack;
 384
 385     /* Setup the processor set */
 386     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 387 }
 388
 389 VOID
 390 NTAPI
 391 INIT_FUNCTION
 392 KiInitializeKernel(IN PKPROCESS InitProcess,
 393                    IN PKTHREAD InitThread,
 394                    IN PVOID IdleStack,
 395                    IN PKPRCB Prcb,
 396                    IN CCHAR Number,
 397                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 398 {
 399     BOOLEAN NpxPresent;
 400     ULONG FeatureBits;
 401     ULONG PageDirectory[2];
 402     PVOID DpcStack;
 403     ULONG Vendor[3];
 404
 405     /* Detect and set the CPU Type */
 406     KiSetProcessorType();
 407
 408     /* Check if an FPU is present */
 409     NpxPresent = KiIsNpxPresent();
 410
 411     /* Initialize the Power Management Support for this PRCB */
 412     PoInitializePrcb(Prcb);
 413
 414     /* Bugcheck if this is a 386 CPU */
 415     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 416
 417     /* Get the processor features for the CPU */
 418     FeatureBits = KiGetFeatureBits();
 419
 420     /* Set the default NX policy (opt-in) */
 421     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 422
 423     /* Check if NPX is always on */
 424     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 425     {
 426         /* Set it always on */
 427         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 428         FeatureBits |= KF_NX_ENABLED;
 429     }
 430     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 431     {
 432         /* Set it in opt-out mode */
 433         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 434         FeatureBits |= KF_NX_ENABLED;
 435     }
 436     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 437              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 438     {
 439         /* Set the feature bits */
 440         FeatureBits |= KF_NX_ENABLED;
 441     }
 442     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 443              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 444     {
 445         /* Set disabled mode */
 446         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 447         FeatureBits |= KF_NX_DISABLED;
 448     }
 449
 450     /* Save feature bits */
 451     Prcb->FeatureBits = FeatureBits;
 452
 453     /* Save CPU state */
 454     KiSaveProcessorControlState(&Prcb->ProcessorState);
 455
 456     /* Get cache line information for this CPU */
 457     KiGetCacheInformation();
 458
 459     /* Initialize spinlocks and DPC data */
 460     KiInitSpinLocks(Prcb, Number);
 461
 462     /* Check if this is the Boot CPU */
 463     if (!Number)
 464     {
 465         /* Set Node Data */
 466         KeNodeBlock[0] = &KiNode0;
 467         Prcb->ParentNode = KeNodeBlock[0];
 468         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 469
 470         /* Set boot-level flags */
 471         KeI386NpxPresent = NpxPresent;
 472         KeI386CpuType = Prcb->CpuType;
 473         KeI386CpuStep = Prcb->CpuStep;
 474         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 475         KeProcessorLevel = (USHORT)Prcb->CpuType;
 476         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 477         KeFeatureBits = FeatureBits;
 478         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 479         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 480
 481         /* Detect 8-byte compare exchange support */
 482         if (!(KeFeatureBits & KF_CMPXCHG8B))
 483         {
 484             /* Copy the vendor string */
 485             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 486
 487             /* Bugcheck the system. Windows *requires* this */
 488             KeBugCheckEx(0x5D,
 489                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 490                          Vendor[0],
 491                          Vendor[1],
 492                          Vendor[2]);
 493         }
 494
 495         /* Set the current MP Master KPRCB to the Boot PRCB */
 496         Prcb->MultiThreadSetMaster = Prcb;
 497
 498         /* Lower to APC_LEVEL */
 499         KeLowerIrql(APC_LEVEL);
 500
 501         /* Initialize some spinlocks */
 502         KeInitializeSpinLock(&KiFreezeExecutionLock);
 503         KeInitializeSpinLock(&Ki486CompatibilityLock);
 504
 505         /* Initialize portable parts of the OS */
 506         KiInitSystem();
 507
 508         /* Initialize the Idle Process and the Process Listhead */
 509         InitializeListHead(&KiProcessListHead);
 510         PageDirectory[0] = 0;
 511         PageDirectory[1] = 0;
 512         KeInitializeProcess(InitProcess,
 513                             0,
 514                             0xFFFFFFFF,
 515                             PageDirectory,
 516                             FALSE);
 517         InitProcess->QuantumReset = MAXCHAR;
 518     }
 519     else
 520     {
 521         /* FIXME */
 522         DPRINT1("SMP Boot support not yet present\n");
 523     }
 524
 525     /* Setup the Idle Thread */
 526     KeInitializeThread(InitProcess,
 527                        InitThread,
 528                        NULL,
 529                        NULL,
 530                        NULL,
 531                        NULL,
 532                        NULL,
 533                        IdleStack);
 534     InitThread->NextProcessor = Number;
 535     InitThread->Priority = HIGH_PRIORITY;
 536     InitThread->State = Running;
 537     InitThread->Affinity = 1 << Number;
 538     InitThread->WaitIrql = DISPATCH_LEVEL;
 539     InitProcess->ActiveProcessors = 1 << Number;
 540
 541     /* HACK for MmUpdatePageDir */
 542     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 543
 544     /* Set basic CPU Features that user mode can read */
 545     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 546         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 547     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 548         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 549     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 550         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 551     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 552         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 553     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 554         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 555     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 556         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 557
 558     /* Set up the thread-related fields in the PRCB */
 559     Prcb->CurrentThread = InitThread;
 560     Prcb->NextThread = NULL;
 561     Prcb->IdleThread = InitThread;
 562
 563     /* Initialize the Kernel Executive */
 564     ExpInitializeExecutive(Number, LoaderBlock);
 565
 566     /* Only do this on the boot CPU */
 567     if (!Number)
 568     {
 569         /* Calculate the time reciprocal */
 570         KiTimeIncrementReciprocal =
 571             KiComputeReciprocal(KeMaximumIncrement,
 572                                 &KiTimeIncrementShiftCount);
 573
 574         /* Update DPC Values in case they got updated by the executive */
 575         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 576         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 577         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 578
 579         /* Allocate the DPC Stack */
 580         DpcStack = MmCreateKernelStack(FALSE, 0);
 581         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 582         Prcb->DpcStack = DpcStack;
 583
 584         /* Allocate the IOPM save area. */
 585         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 586                                                   PAGE_SIZE * 2,
 587                                                   '  eK');
 588         if (!Ki386IopmSaveArea)
 589         {
 590             /* Bugcheck. We need this for V86/VDM support. */
 591             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 592         }
 593     }
 594
 595     /* Raise to Dispatch */
 596     KfRaiseIrql(DISPATCH_LEVEL);
 597
 598     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 599     KeSetPriorityThread(InitThread, 0);
 600
 601     /* If there's no thread scheduled, put this CPU in the Idle summary */
 602     KiAcquirePrcbLock(Prcb);
 603     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 604     KiReleasePrcbLock(Prcb);
 605
 606     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 607     KfRaiseIrql(HIGH_LEVEL);
 608     LoaderBlock->Prcb = 0;
 609 }
 610
 611 VOID
 612 FASTCALL
 613 INIT_FUNCTION
 614 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 615                          IN PKIDTENTRY *Idt,
 616                          IN PKIPCR *Pcr,
 617                          IN PKTSS *Tss)
 618 {
 619     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 620     KGDTENTRY TssSelector, PcrSelector;
 621     USHORT Tr, Fs;
 622
 623     /* Get GDT and IDT descriptors */
 624     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 625     __sidt(&IdtDescriptor.Limit);
 626
 627     /* Save IDT and GDT */
 628     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 629     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 630
 631     /* Get TSS and FS Selectors */
 632     Tr = Ke386GetTr();
 633     Fs = Ke386GetFs();
 634
 635     /* Get PCR Selector, mask it and get its GDT Entry */
 636     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 637
 638     /* Get the KPCR itself */
 639     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 640                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 641                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 642
 643     /* Get TSS Selector, mask it and get its GDT Entry */
 644     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 645
 646     /* Get the KTSS itself */
 647     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 648                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 649                               TssSelector.HighWord.Bytes.BaseHi << 24);
 650 }
 651
 652 VOID
 653 NTAPI
 654 INIT_FUNCTION
 655 KiSystemStartupBootStack(VOID)
 656 {
 657     PKTHREAD Thread;
 658
 659     /* Initialize the kernel for the current CPU */
 660     KiInitializeKernel(&KiInitialProcess.Pcb,
 661                        (PKTHREAD)KeLoaderBlock->Thread,
 662                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 663                        (PKPRCB)__readfsdword(KPCR_PRCB),
 664                        KeNumberProcessors - 1,
 665                        KeLoaderBlock);
 666
 667     /* Set the priority of this thread to 0 */
 668     Thread = KeGetCurrentThread();
 669     Thread->Priority = 0;
 670
 671     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 672     _enable();
 673     KfLowerIrql(DISPATCH_LEVEL);
 674
 675     /* Set the right wait IRQL */
 676     Thread->WaitIrql = DISPATCH_LEVEL;
 677
 678     /* Jump into the idle loop */
 679     KiIdleLoop();
 680 }
 681
 682 VOID
 683 NTAPI
 684 INIT_FUNCTION
 685 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 686 {
 687     ULONG Cpu;
 688     PKTHREAD InitialThread;
 689     ULONG InitialStack;
 690     PKGDTENTRY Gdt;
 691     PKIDTENTRY Idt;
 692     KIDTENTRY NmiEntry, DoubleFaultEntry;
 693     PKTSS Tss;
 694     PKIPCR Pcr;
 695
 696     /* Boot cycles timestamp */
 697     BootCycles = __rdtsc();
 698 #if !defined(_X86_)
 699     /* Check if we are being booted from FreeLDR */
 700     if (!((ULONG_PTR)LoaderBlock & 0x80000000)) KiRosPrepareForSystemStartup((PROS_LOADER_PARAMETER_BLOCK)LoaderBlock);
 701 #endif
 702     /* Save the loader block and get the current CPU */
 703     KeLoaderBlock = LoaderBlock;
 704     Cpu = KeNumberProcessors;
 705     if (!Cpu)
 706     {
 707         /* If this is the boot CPU, set FS and the CPU Number*/
 708         Ke386SetFs(KGDT_R0_PCR);
 709         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 710
 711         /* Set the initial stack and idle thread as well */
 712         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 713         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 714     }
 715
 716     /* Save the initial thread and stack */
 717     InitialStack = LoaderBlock->KernelStack;
 718     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 719
 720     /* Clean the APC List Head */
 721     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 722
 723     /* Initialize the machine type */
 724     KiInitializeMachineType();
 725
 726     /* Skip initial setup if this isn't the Boot CPU */
 727     if (Cpu) goto AppCpuInit;
 728
 729     /* Get GDT, IDT, PCR and TSS pointers */
 730     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 731
 732     /* Setup the TSS descriptors and entries */
 733     Ki386InitializeTss(Tss, Idt, Gdt);
 734
 735     /* Initialize the PCR */
 736     RtlZeroMemory(Pcr, PAGE_SIZE);
 737     KiInitializePcr(Cpu,
 738                     Pcr,
 739                     Idt,
 740                     Gdt,
 741                     Tss,
 742                     InitialThread,
 743                     (PVOID)KiDoubleFaultStack);
 744
 745     /* Set us as the current process */
 746     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 747
 748     /* Clear DR6/7 to cleanup bootloader debugging */
 749     __writefsdword(KPCR_TEB, 0);
 750     __writefsdword(KPCR_DR6, 0);
 751     __writefsdword(KPCR_DR7, 0);
 752
 753     /* Setup the IDT */
 754     KeInitExceptions();
 755
 756     /* Load Ring 3 selectors for DS/ES */
 757     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 758     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 759
 760     /* Save NMI and double fault traps */
 761     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 762     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 763
 764     /* Copy kernel's trap handlers */
 765     RtlCopyMemory(Idt,
 766                   (PVOID)KiIdtDescriptor.Base,
 767                   KiIdtDescriptor.Limit + 1);
 768
 769     /* Restore NMI and double fault */
 770     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 771     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 772
 773 AppCpuInit:
 774     /* Loop until we can release the freeze lock */
 775     do
 776     {
 777         /* Loop until execution can continue */
 778         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 779     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 780
 781     /* Setup CPU-related fields */
 782     __writefsdword(KPCR_NUMBER, Cpu);
 783     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 784     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 785     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 786
 787     /* Initialize the Processor with HAL */
 788     HalInitializeProcessor(Cpu, KeLoaderBlock);
 789
 790     /* Set active processors */
 791     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 792     KeNumberProcessors++;
 793
 794     /* Check if this is the boot CPU */
 795     if (!Cpu)
 796     {
 797         /* Initialize debugging system */
 798         KdInitSystem(0, KeLoaderBlock);
 799
 800         /* Check for break-in */
 801         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 802     }
 803
 804     /* Raise to HIGH_LEVEL */
 805     KfRaiseIrql(HIGH_LEVEL);
 806
 807     /* Switch to new kernel stack and start kernel bootstrapping */
 808     KiSwitchToBootStack(InitialStack & ~3);
 809 }