reactos/ntoskrnl/ke/i386/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES *****************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14 #include "internal/trap_x.h"
  15
  16 /* GLOBALS *******************************************************************/
  17
  18 /* Boot and double-fault/NMI/DPC stack */
  19 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  20 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  23
  24 /* Spinlocks used only on X86 */
  25 KSPIN_LOCK KiFreezeExecutionLock;
  26 KSPIN_LOCK Ki486CompatibilityLock;
  27
  28 /* Perf */
  29 ULONG ProcessCount;
  30 ULONGLONG BootCycles, BootCyclesEnd;
  31
  32 /* FUNCTIONS *****************************************************************/
  33
  34 VOID
  35 NTAPI
  36 INIT_FUNCTION
  37 KiInitMachineDependent(VOID)
  38 {
  39     ULONG CpuCount;
  40     BOOLEAN FbCaching = FALSE;
  41     NTSTATUS Status;
  42     ULONG ReturnLength;
  43     ULONG i, Affinity, Sample = 0;
  44     PFX_SAVE_AREA FxSaveArea;
  45     ULONG MXCsrMask = 0xFFBF;
  46     ULONG Dummy;
  47     KI_SAMPLE_MAP Samples[4];
  48     PKI_SAMPLE_MAP CurrentSample = Samples;
  49
  50     /* Check for large page support */
  51     if (KeFeatureBits & KF_LARGE_PAGE)
  52     {
  53         /* FIXME: Support this */
  54         DPRINT1("Large Page support detected but not yet taken advantage of\n");
  55     }
  56
  57     /* Check for global page support */
  58     if (KeFeatureBits & KF_GLOBAL_PAGE)
  59     {
  60         /* Do an IPI to enable it on all CPUs */
  61         CpuCount = KeNumberProcessors;
  62         KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
  63     }
  64
  65     /* Check for PAT and/or MTRR support */
  66     if (KeFeatureBits & (KF_PAT | KF_MTRR))
  67     {
  68         /* Query the HAL to make sure we can use it */
  69         Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
  70                                            sizeof(BOOLEAN),
  71                                            &FbCaching,
  72                                            &ReturnLength);
  73         if ((NT_SUCCESS(Status)) && (FbCaching))
  74         {
  75             /* We can't, disable it */
  76             KeFeatureBits &= ~(KF_PAT | KF_MTRR);
  77         }
  78     }
  79
  80     /* Check for PAT support and enable it */
  81     if (KeFeatureBits & KF_PAT) KiInitializePAT();
  82
  83     /* Assume no errata for now */
  84     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
  85
  86     /* Check if we have an NPX */
  87     if (KeI386NpxPresent)
  88     {
  89         /* Loop every CPU */
  90         i = KeActiveProcessors;
  91         for (Affinity = 1; i; Affinity <<= 1)
  92         {
  93             /* Check if this is part of the set */
  94             if (i & Affinity)
  95             {
  96                 /* Run on this CPU */
  97                 i &= ~Affinity;
  98                 KeSetSystemAffinityThread(Affinity);
  99
 100                 /* Detect FPU errata */
 101                 if (KiIsNpxErrataPresent())
 102                 {
 103                     /* Disable NPX support */
 104                     KeI386NpxPresent = FALSE;
 105                     SharedUserData->
 106                         ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 107                         TRUE;
 108                     break;
 109                 }
 110             }
 111         }
 112     }
 113
 114     /* If there's no NPX, then we're emulating the FPU */
 115     SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
 116         !KeI386NpxPresent;
 117
 118     /* Check if there's no NPX, so that we can disable associated features */
 119     if (!KeI386NpxPresent)
 120     {
 121         /* Remove NPX-related bits */
 122         KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
 123
 124         /* Disable kernel flags */
 125         KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
 126
 127         /* Disable processor features that might've been set until now */
 128         SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
 129         SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE]   =
 130         SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE]     =
 131         SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE]    =
 132         SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
 133     }
 134
 135     /* Check for CR4 support */
 136     if (KeFeatureBits & KF_CR4)
 137     {
 138         /* Do an IPI call to enable the Debug Exceptions */
 139         CpuCount = KeNumberProcessors;
 140         KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
 141     }
 142
 143     /* Check if FXSR was found */
 144     if (KeFeatureBits & KF_FXSR)
 145     {
 146         /* Do an IPI call to enable the FXSR */
 147         CpuCount = KeNumberProcessors;
 148         KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
 149
 150         /* Check if XMM was found too */
 151         if (KeFeatureBits & KF_XMMI)
 152         {
 153             /* Do an IPI call to enable XMMI exceptions */
 154             CpuCount = KeNumberProcessors;
 155             KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
 156
 157             /* FIXME: Implement and enable XMM Page Zeroing for Mm */
 158
 159             /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
 160             *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
 161         }
 162     }
 163
 164     /* Check for, and enable SYSENTER support */
 165     KiRestoreFastSyscallReturnState();
 166
 167     /* Loop every CPU */
 168     i = KeActiveProcessors;
 169     for (Affinity = 1; i; Affinity <<= 1)
 170     {
 171         /* Check if this is part of the set */
 172         if (i & Affinity)
 173         {
 174             /* Run on this CPU */
 175             i &= ~Affinity;
 176             KeSetSystemAffinityThread(Affinity);
 177
 178             /* Reset MHz to 0 for this CPU */
 179             KeGetCurrentPrcb()->MHz = 0;
 180
 181             /* Check if we can use RDTSC */
 182             if (KeFeatureBits & KF_RDTSC)
 183             {
 184                 /* Start sampling loop */
 185                 for (;;)
 186                 {
 187                     /* Do a dummy CPUID to start the sample */
 188                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 189
 190                     /* Fill out the starting data */
 191                     CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
 192                     CurrentSample->TSCStart = __rdtsc();
 193                     CurrentSample->PerfFreq.QuadPart = -50000;
 194
 195                     /* Sleep for this sample */
 196                     KeDelayExecutionThread(KernelMode,
 197                                            FALSE,
 198                                            &CurrentSample->PerfFreq);
 199
 200                     /* Do another dummy CPUID */
 201                     CPUID(0, &Dummy, &Dummy, &Dummy, &Dummy);
 202
 203                     /* Fill out the ending data */
 204                     CurrentSample->PerfEnd =
 205                         KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
 206                     CurrentSample->TSCEnd = __rdtsc();
 207
 208                     /* Calculate the differences */
 209                     CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
 210                                                CurrentSample->PerfStart.QuadPart;
 211                     CurrentSample->TSCDelta = CurrentSample->TSCEnd -
 212                                               CurrentSample->TSCStart;
 213
 214                     /* Compute CPU Speed */
 215                     CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
 216                                                   CurrentSample->
 217                                                   PerfFreq.QuadPart + 500000) /
 218                                                  (CurrentSample->PerfDelta *
 219                                                   1000000));
 220
 221                     /* Check if this isn't the first sample */
 222                     if (Sample)
 223                     {
 224                         /* Check if we got a good precision within 1MHz */
 225                         if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
 226                             (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
 227                             (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
 228                         {
 229                             /* We did, stop sampling */
 230                             break;
 231                         }
 232                     }
 233
 234                     /* Move on */
 235                     CurrentSample++;
 236                     Sample++;
 237
 238                     if (Sample == sizeof(Samples) / sizeof(Samples[0]))
 239                     {
 240                         /* Restart */
 241                         CurrentSample = Samples;
 242                         Sample = 0;
 243                     }
 244                 }
 245
 246                 /* Save the CPU Speed */
 247                 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
 248             }
 249
 250             /* Check if we have MTRR */
 251             if (KeFeatureBits & KF_MTRR)
 252             {
 253                 /* Then manually initialize MTRR for the CPU */
 254                 KiInitializeMTRR(i ? FALSE : TRUE);
 255             }
 256
 257             /* Check if we have AMD MTRR and initialize it for the CPU */
 258             if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
 259
 260             /* Check if this is a buggy Pentium and apply the fixup if so */
 261             if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
 262
 263             /* Check if the CPU supports FXSR */
 264             if (KeFeatureBits & KF_FXSR)
 265             {
 266                 /* Get the current thread NPX state */
 267                 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
 268
 269                 /* Clear initial MXCsr mask */
 270                 FxSaveArea->U.FxArea.MXCsrMask = 0;
 271
 272                 /* Save the current NPX State */
 273                 Ke386SaveFpuState(FxSaveArea);
 274
 275                 /* Check if the current mask doesn't match the reserved bits */
 276                 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
 277                 {
 278                     /* Then use whatever it's holding */
 279                     MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
 280                 }
 281
 282                 /* Check if nobody set the kernel-wide mask */
 283                 if (!KiMXCsrMask)
 284                 {
 285                     /* Then use the one we calculated above */
 286                     KiMXCsrMask = MXCsrMask;
 287                 }
 288                 else
 289                 {
 290                     /* Was it set to the same value we found now? */
 291                     if (KiMXCsrMask != MXCsrMask)
 292                     {
 293                         /* No, something is definitely wrong */
 294                         KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
 295                                      KF_FXSR,
 296                                      KiMXCsrMask,
 297                                      MXCsrMask,
 298                                      0);
 299                     }
 300                 }
 301
 302                 /* Now set the kernel mask */
 303                 KiMXCsrMask &= MXCsrMask;
 304             }
 305         }
 306     }
 307
 308     /* Return affinity back to where it was */
 309     KeRevertToUserAffinityThread();
 310
 311     /* NT allows limiting the duration of an ISR with a registry key */
 312     if (KiTimeLimitIsrMicroseconds)
 313     {
 314         /* FIXME: TODO */
 315         DPRINT1("ISR Time Limit not yet supported\n");
 316     }
 317
 318     /* Set CR0 features based on detected CPU */
 319     KiSetCR0Bits();
 320 }
 321
 322 VOID
 323 NTAPI
 324 INIT_FUNCTION
 325 KiInitializePcr(IN ULONG ProcessorNumber,
 326                 IN PKIPCR Pcr,
 327                 IN PKIDTENTRY Idt,
 328                 IN PKGDTENTRY Gdt,
 329                 IN PKTSS Tss,
 330                 IN PKTHREAD IdleThread,
 331                 IN PVOID DpcStack)
 332 {
 333     /* Setup the TIB */
 334     Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
 335     Pcr->NtTib.StackBase = 0;
 336     Pcr->NtTib.StackLimit = 0;
 337     Pcr->NtTib.Self = NULL;
 338
 339     /* Set the Current Thread */
 340     Pcr->PrcbData.CurrentThread = IdleThread;
 341
 342     /* Set pointers to ourselves */
 343     Pcr->Self = (PKPCR)Pcr;
 344     Pcr->Prcb = &Pcr->PrcbData;
 345
 346     /* Set the PCR Version */
 347     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 348     Pcr->MinorVersion = PCR_MINOR_VERSION;
 349
 350     /* Set the PCRB Version */
 351     Pcr->PrcbData.MajorVersion = 1;
 352     Pcr->PrcbData.MinorVersion = 1;
 353
 354     /* Set the Build Type */
 355     Pcr->PrcbData.BuildType = 0;
 356 #ifndef CONFIG_SMP
 357     Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 358 #endif
 359 #if DBG
 360     Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
 361 #endif
 362
 363     /* Set the Processor Number and current Processor Mask */
 364     Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
 365     Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
 366
 367     /* Set the PRCB for this Processor */
 368     KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
 369
 370     /* Start us out at PASSIVE_LEVEL */
 371     Pcr->Irql = PASSIVE_LEVEL;
 372
 373     /* Set the GDI, IDT, TSS and DPC Stack */
 374     Pcr->GDT = (PVOID)Gdt;
 375     Pcr->IDT = Idt;
 376     Pcr->TSS = Tss;
 377     Pcr->TssCopy = Tss;
 378     Pcr->PrcbData.DpcStack = DpcStack;
 379
 380     /* Setup the processor set */
 381     Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
 382 }
 383
 384 VOID
 385 NTAPI
 386 INIT_FUNCTION
 387 KiInitializeKernel(IN PKPROCESS InitProcess,
 388                    IN PKTHREAD InitThread,
 389                    IN PVOID IdleStack,
 390                    IN PKPRCB Prcb,
 391                    IN CCHAR Number,
 392                    IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 393 {
 394     BOOLEAN NpxPresent;
 395     ULONG FeatureBits;
 396     ULONG PageDirectory[2];
 397     PVOID DpcStack;
 398     ULONG Vendor[3];
 399
 400     /* Detect and set the CPU Type */
 401     KiSetProcessorType();
 402
 403     /* Check if an FPU is present */
 404     NpxPresent = KiIsNpxPresent();
 405
 406     /* Initialize the Power Management Support for this PRCB */
 407     PoInitializePrcb(Prcb);
 408
 409     /* Bugcheck if this is a 386 CPU */
 410     if (Prcb->CpuType == 3) KeBugCheckEx(0x5D, 0x386, 0, 0, 0);
 411
 412     /* Get the processor features for the CPU */
 413     FeatureBits = KiGetFeatureBits();
 414
 415     /* Set the default NX policy (opt-in) */
 416     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 417
 418     /* Check if NPX is always on */
 419     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 420     {
 421         /* Set it always on */
 422         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 423         FeatureBits |= KF_NX_ENABLED;
 424     }
 425     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 426     {
 427         /* Set it in opt-out mode */
 428         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 429         FeatureBits |= KF_NX_ENABLED;
 430     }
 431     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 432              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 433     {
 434         /* Set the feature bits */
 435         FeatureBits |= KF_NX_ENABLED;
 436     }
 437     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 438              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 439     {
 440         /* Set disabled mode */
 441         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 442         FeatureBits |= KF_NX_DISABLED;
 443     }
 444
 445     /* Save feature bits */
 446     Prcb->FeatureBits = FeatureBits;
 447
 448     /* Save CPU state */
 449     KiSaveProcessorControlState(&Prcb->ProcessorState);
 450
 451     /* Get cache line information for this CPU */
 452     KiGetCacheInformation();
 453
 454     /* Initialize spinlocks and DPC data */
 455     KiInitSpinLocks(Prcb, Number);
 456
 457     /* Check if this is the Boot CPU */
 458     if (!Number)
 459     {
 460         /* Set Node Data */
 461         KeNodeBlock[0] = &KiNode0;
 462         Prcb->ParentNode = KeNodeBlock[0];
 463         KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
 464
 465         /* Set boot-level flags */
 466         KeI386NpxPresent = NpxPresent;
 467         KeI386CpuType = Prcb->CpuType;
 468         KeI386CpuStep = Prcb->CpuStep;
 469         KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
 470         KeProcessorLevel = (USHORT)Prcb->CpuType;
 471         if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 472         KeFeatureBits = FeatureBits;
 473         KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
 474         KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
 475
 476         /* Detect 8-byte compare exchange support */
 477         if (!(KeFeatureBits & KF_CMPXCHG8B))
 478         {
 479             /* Copy the vendor string */
 480             RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
 481
 482             /* Bugcheck the system. Windows *requires* this */
 483             KeBugCheckEx(0x5D,
 484                          (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
 485                          Vendor[0],
 486                          Vendor[1],
 487                          Vendor[2]);
 488         }
 489
 490         /* Set the current MP Master KPRCB to the Boot PRCB */
 491         Prcb->MultiThreadSetMaster = Prcb;
 492
 493         /* Lower to APC_LEVEL */
 494         KeLowerIrql(APC_LEVEL);
 495
 496         /* Initialize some spinlocks */
 497         KeInitializeSpinLock(&KiFreezeExecutionLock);
 498         KeInitializeSpinLock(&Ki486CompatibilityLock);
 499
 500         /* Initialize portable parts of the OS */
 501         KiInitSystem();
 502
 503         /* Initialize the Idle Process and the Process Listhead */
 504         InitializeListHead(&KiProcessListHead);
 505         PageDirectory[0] = 0;
 506         PageDirectory[1] = 0;
 507         KeInitializeProcess(InitProcess,
 508                             0,
 509                             0xFFFFFFFF,
 510                             PageDirectory,
 511                             FALSE);
 512         InitProcess->QuantumReset = MAXCHAR;
 513     }
 514     else
 515     {
 516         /* FIXME */
 517         DPRINT1("SMP Boot support not yet present\n");
 518     }
 519
 520     /* Setup the Idle Thread */
 521     KeInitializeThread(InitProcess,
 522                        InitThread,
 523                        NULL,
 524                        NULL,
 525                        NULL,
 526                        NULL,
 527                        NULL,
 528                        IdleStack);
 529     InitThread->NextProcessor = Number;
 530     InitThread->Priority = HIGH_PRIORITY;
 531     InitThread->State = Running;
 532     InitThread->Affinity = 1 << Number;
 533     InitThread->WaitIrql = DISPATCH_LEVEL;
 534     InitProcess->ActiveProcessors = 1 << Number;
 535
 536     /* HACK for MmUpdatePageDir */
 537     ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
 538
 539     /* Set basic CPU Features that user mode can read */
 540     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 541         (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
 542     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
 543         (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
 544     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 545         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
 546     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 547         ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 548     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 549         (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
 550     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
 551         (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
 552
 553     /* Set up the thread-related fields in the PRCB */
 554     Prcb->CurrentThread = InitThread;
 555     Prcb->NextThread = NULL;
 556     Prcb->IdleThread = InitThread;
 557
 558     /* Initialize the Kernel Executive */
 559     ExpInitializeExecutive(Number, LoaderBlock);
 560
 561     /* Only do this on the boot CPU */
 562     if (!Number)
 563     {
 564         /* Calculate the time reciprocal */
 565         KiTimeIncrementReciprocal =
 566             KiComputeReciprocal(KeMaximumIncrement,
 567                                 &KiTimeIncrementShiftCount);
 568
 569         /* Update DPC Values in case they got updated by the executive */
 570         Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
 571         Prcb->MinimumDpcRate = KiMinimumDpcRate;
 572         Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
 573
 574         /* Allocate the DPC Stack */
 575         DpcStack = MmCreateKernelStack(FALSE, 0);
 576         if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
 577         Prcb->DpcStack = DpcStack;
 578
 579         /* Allocate the IOPM save area. */
 580         Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
 581                                                   PAGE_SIZE * 2,
 582                                                   '  eK');
 583         if (!Ki386IopmSaveArea)
 584         {
 585             /* Bugcheck. We need this for V86/VDM support. */
 586             KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
 587         }
 588     }
 589
 590     /* Raise to Dispatch */
 591     KfRaiseIrql(DISPATCH_LEVEL);
 592
 593     /* Set the Idle Priority to 0. This will jump into Phase 1 */
 594     KeSetPriorityThread(InitThread, 0);
 595
 596     /* If there's no thread scheduled, put this CPU in the Idle summary */
 597     KiAcquirePrcbLock(Prcb);
 598     if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
 599     KiReleasePrcbLock(Prcb);
 600
 601     /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
 602     KfRaiseIrql(HIGH_LEVEL);
 603     LoaderBlock->Prcb = 0;
 604 }
 605
 606 VOID
 607 FASTCALL
 608 INIT_FUNCTION
 609 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
 610                          IN PKIDTENTRY *Idt,
 611                          IN PKIPCR *Pcr,
 612                          IN PKTSS *Tss)
 613 {
 614     KDESCRIPTOR GdtDescriptor, IdtDescriptor;
 615     KGDTENTRY TssSelector, PcrSelector;
 616     USHORT Tr, Fs;
 617
 618     /* Get GDT and IDT descriptors */
 619     Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
 620     __sidt(&IdtDescriptor.Limit);
 621
 622     /* Save IDT and GDT */
 623     *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
 624     *Idt = (PKIDTENTRY)IdtDescriptor.Base;
 625
 626     /* Get TSS and FS Selectors */
 627     Tr = Ke386GetTr();
 628     Fs = Ke386GetFs();
 629
 630     /* Get PCR Selector, mask it and get its GDT Entry */
 631     PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
 632
 633     /* Get the KPCR itself */
 634     *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
 635                                PcrSelector.HighWord.Bytes.BaseMid << 16 |
 636                                PcrSelector.HighWord.Bytes.BaseHi << 24);
 637
 638     /* Get TSS Selector, mask it and get its GDT Entry */
 639     TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
 640
 641     /* Get the KTSS itself */
 642     *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
 643                               TssSelector.HighWord.Bytes.BaseMid << 16 |
 644                               TssSelector.HighWord.Bytes.BaseHi << 24);
 645 }
 646
 647 VOID
 648 NTAPI
 649 INIT_FUNCTION
 650 KiSystemStartupBootStack(VOID)
 651 {
 652     PKTHREAD Thread;
 653
 654     /* Initialize the kernel for the current CPU */
 655     KiInitializeKernel(&KiInitialProcess.Pcb,
 656                        (PKTHREAD)KeLoaderBlock->Thread,
 657                        (PVOID)(KeLoaderBlock->KernelStack & ~3),
 658                        (PKPRCB)__readfsdword(KPCR_PRCB),
 659                        KeNumberProcessors - 1,
 660                        KeLoaderBlock);
 661
 662     /* Set the priority of this thread to 0 */
 663     Thread = KeGetCurrentThread();
 664     Thread->Priority = 0;
 665
 666     /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
 667     _enable();
 668     KfLowerIrql(DISPATCH_LEVEL);
 669
 670     /* Set the right wait IRQL */
 671     Thread->WaitIrql = DISPATCH_LEVEL;
 672
 673     /* Jump into the idle loop */
 674     KiIdleLoop();
 675 }
 676
 677 VOID
 678 NTAPI
 679 INIT_FUNCTION
 680 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 681 {
 682     ULONG Cpu;
 683     PKTHREAD InitialThread;
 684     ULONG InitialStack;
 685     PKGDTENTRY Gdt;
 686     PKIDTENTRY Idt;
 687     KIDTENTRY NmiEntry, DoubleFaultEntry;
 688     PKTSS Tss;
 689     PKIPCR Pcr;
 690
 691     /* Boot cycles timestamp */
 692     BootCycles = __rdtsc();
 693 #if !defined(_X86_)
 694     /* Check if we are being booted from FreeLDR */
 695     if (!((ULONG_PTR)LoaderBlock & 0x80000000)) KiRosPrepareForSystemStartup((PROS_LOADER_PARAMETER_BLOCK)LoaderBlock);
 696 #endif
 697     /* Save the loader block and get the current CPU */
 698     KeLoaderBlock = LoaderBlock;
 699     Cpu = KeNumberProcessors;
 700     if (!Cpu)
 701     {
 702         /* If this is the boot CPU, set FS and the CPU Number*/
 703         Ke386SetFs(KGDT_R0_PCR);
 704         __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
 705
 706         /* Set the initial stack and idle thread as well */
 707         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 708         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 709     }
 710
 711     /* Save the initial thread and stack */
 712     InitialStack = LoaderBlock->KernelStack;
 713     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 714
 715     /* Clean the APC List Head */
 716     InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
 717
 718     /* Initialize the machine type */
 719     KiInitializeMachineType();
 720
 721     /* Skip initial setup if this isn't the Boot CPU */
 722     if (Cpu) goto AppCpuInit;
 723
 724     /* Get GDT, IDT, PCR and TSS pointers */
 725     KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
 726
 727     /* Setup the TSS descriptors and entries */
 728     Ki386InitializeTss(Tss, Idt, Gdt);
 729
 730     /* Initialize the PCR */
 731     RtlZeroMemory(Pcr, PAGE_SIZE);
 732     KiInitializePcr(Cpu,
 733                     Pcr,
 734                     Idt,
 735                     Gdt,
 736                     Tss,
 737                     InitialThread,
 738                     (PVOID)KiDoubleFaultStack);
 739
 740     /* Set us as the current process */
 741     InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
 742
 743     /* Clear DR6/7 to cleanup bootloader debugging */
 744     __writefsdword(KPCR_TEB, 0);
 745     __writefsdword(KPCR_DR6, 0);
 746     __writefsdword(KPCR_DR7, 0);
 747
 748     /* Setup the IDT */
 749     KeInitExceptions();
 750
 751     /* Load Ring 3 selectors for DS/ES */
 752     Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
 753     Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
 754
 755     /* Save NMI and double fault traps */
 756     RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
 757     RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
 758
 759     /* Copy kernel's trap handlers */
 760     RtlCopyMemory(Idt,
 761                   (PVOID)KiIdtDescriptor.Base,
 762                   KiIdtDescriptor.Limit + 1);
 763
 764     /* Restore NMI and double fault */
 765     RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
 766     RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
 767
 768 AppCpuInit:
 769     /* Loop until we can release the freeze lock */
 770     do
 771     {
 772         /* Loop until execution can continue */
 773         while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
 774     } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
 775
 776     /* Setup CPU-related fields */
 777     __writefsdword(KPCR_NUMBER, Cpu);
 778     __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
 779     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
 780     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 781
 782     /* Initialize the Processor with HAL */
 783     HalInitializeProcessor(Cpu, KeLoaderBlock);
 784
 785     /* Set active processors */
 786     KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
 787     KeNumberProcessors++;
 788
 789     /* Check if this is the boot CPU */
 790     if (!Cpu)
 791     {
 792         /* Initialize debugging system */
 793         KdInitSystem(0, KeLoaderBlock);
 794
 795         /* Check for break-in */
 796         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 797     }
 798
 799     /* Raise to HIGH_LEVEL */
 800     KfRaiseIrql(HIGH_LEVEL);
 801
 802     /* Switch to new kernel stack and start kernel bootstrapping */
 803     KiSwitchToBootStack(InitialStack & ~3);
 804 }