cba2c423fd4b2b92d3c09beb3c95d22030638841
[reactos.git] / ntoskrnl / ke / i386 / kiinit.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14 #include "internal/i386/trap_x.h"
15
16 /* GLOBALS *******************************************************************/
17
18 /* Boot and double-fault/NMI/DPC stack */
19 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) P0BootStackData[KERNEL_STACK_SIZE] = {0};
20 UCHAR DECLSPEC_ALIGN(PAGE_SIZE) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
21 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
22 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
23
24 /* Spinlocks used only on X86 */
25 KSPIN_LOCK KiFreezeExecutionLock;
26 KSPIN_LOCK Ki486CompatibilityLock;
27
28 /* Perf */
29 ULONG ProcessCount;
30 ULONGLONG BootCycles, BootCyclesEnd;
31
32 /* FUNCTIONS *****************************************************************/
33
34 INIT_SECTION
35 VOID
36 NTAPI
37 KiInitMachineDependent(VOID)
38 {
39 ULONG CpuCount;
40 BOOLEAN FbCaching = FALSE;
41 NTSTATUS Status;
42 ULONG ReturnLength;
43 ULONG i, Affinity, Sample = 0;
44 PFX_SAVE_AREA FxSaveArea;
45 ULONG MXCsrMask = 0xFFBF;
46 CPU_INFO CpuInfo;
47 KI_SAMPLE_MAP Samples[4];
48 PKI_SAMPLE_MAP CurrentSample = Samples;
49 LARGE_IDENTITY_MAP IdentityMap;
50
51 /* Check for large page support */
52 if (KeFeatureBits & KF_LARGE_PAGE)
53 {
54 /* Do an IPI to enable it on all CPUs */
55 if (Ki386CreateIdentityMap(&IdentityMap, Ki386EnableCurrentLargePage, 2))
56 KeIpiGenericCall(Ki386EnableTargetLargePage, (ULONG_PTR)&IdentityMap);
57
58 /* Free the pages allocated for identity map */
59 Ki386FreeIdentityMap(&IdentityMap);
60 }
61
62 /* Check for global page support */
63 if (KeFeatureBits & KF_GLOBAL_PAGE)
64 {
65 /* Do an IPI to enable it on all CPUs */
66 CpuCount = KeNumberProcessors;
67 KeIpiGenericCall(Ki386EnableGlobalPage, (ULONG_PTR)&CpuCount);
68 }
69
70 /* Check for PAT and/or MTRR support */
71 if (KeFeatureBits & (KF_PAT | KF_MTRR))
72 {
73 /* Query the HAL to make sure we can use it */
74 Status = HalQuerySystemInformation(HalFrameBufferCachingInformation,
75 sizeof(BOOLEAN),
76 &FbCaching,
77 &ReturnLength);
78 if ((NT_SUCCESS(Status)) && (FbCaching))
79 {
80 /* We can't, disable it */
81 KeFeatureBits &= ~(KF_PAT | KF_MTRR);
82 }
83 }
84
85 /* Check for PAT support and enable it */
86 if (KeFeatureBits & KF_PAT) KiInitializePAT();
87
88 /* Assume no errata for now */
89 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = 0;
90
91 /* Check if we have an NPX */
92 if (KeI386NpxPresent)
93 {
94 /* Loop every CPU */
95 i = KeActiveProcessors;
96 for (Affinity = 1; i; Affinity <<= 1)
97 {
98 /* Check if this is part of the set */
99 if (i & Affinity)
100 {
101 /* Run on this CPU */
102 i &= ~Affinity;
103 KeSetSystemAffinityThread(Affinity);
104
105 /* Detect FPU errata */
106 if (KiIsNpxErrataPresent())
107 {
108 /* Disable NPX support */
109 KeI386NpxPresent = FALSE;
110 SharedUserData->
111 ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
112 TRUE;
113 break;
114 }
115 }
116 }
117 }
118
119 /* If there's no NPX, then we're emulating the FPU */
120 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] =
121 !KeI386NpxPresent;
122
123 /* Check if there's no NPX, so that we can disable associated features */
124 if (!KeI386NpxPresent)
125 {
126 /* Remove NPX-related bits */
127 KeFeatureBits &= ~(KF_XMMI64 | KF_XMMI | KF_FXSR | KF_MMX);
128
129 /* Disable kernel flags */
130 KeI386FxsrPresent = KeI386XMMIPresent = FALSE;
131
132 /* Disable processor features that might've been set until now */
133 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] =
134 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
135 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
136 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
137 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] = 0;
138 }
139
140 /* Check for CR4 support */
141 if (KeFeatureBits & KF_CR4)
142 {
143 /* Do an IPI call to enable the Debug Exceptions */
144 CpuCount = KeNumberProcessors;
145 KeIpiGenericCall(Ki386EnableDE, (ULONG_PTR)&CpuCount);
146 }
147
148 /* Check if FXSR was found */
149 if (KeFeatureBits & KF_FXSR)
150 {
151 /* Do an IPI call to enable the FXSR */
152 CpuCount = KeNumberProcessors;
153 KeIpiGenericCall(Ki386EnableFxsr, (ULONG_PTR)&CpuCount);
154
155 /* Check if XMM was found too */
156 if (KeFeatureBits & KF_XMMI)
157 {
158 /* Do an IPI call to enable XMMI exceptions */
159 CpuCount = KeNumberProcessors;
160 KeIpiGenericCall(Ki386EnableXMMIExceptions, (ULONG_PTR)&CpuCount);
161
162 /* FIXME: Implement and enable XMM Page Zeroing for Mm */
163
164 /* Patch the RtlPrefetchMemoryNonTemporal routine to enable it */
165 *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90; // NOP
166 }
167 }
168
169 /* Check for, and enable SYSENTER support */
170 KiRestoreFastSyscallReturnState();
171
172 /* Loop every CPU */
173 i = KeActiveProcessors;
174 for (Affinity = 1; i; Affinity <<= 1)
175 {
176 /* Check if this is part of the set */
177 if (i & Affinity)
178 {
179 /* Run on this CPU */
180 i &= ~Affinity;
181 KeSetSystemAffinityThread(Affinity);
182
183 /* Reset MHz to 0 for this CPU */
184 KeGetCurrentPrcb()->MHz = 0;
185
186 /* Check if we can use RDTSC */
187 if (KeFeatureBits & KF_RDTSC)
188 {
189 /* Start sampling loop */
190 for (;;)
191 {
192 /* Do a dummy CPUID to start the sample */
193 KiCpuId(&CpuInfo, 0);
194
195 /* Fill out the starting data */
196 CurrentSample->PerfStart = KeQueryPerformanceCounter(NULL);
197 CurrentSample->TSCStart = __rdtsc();
198 CurrentSample->PerfFreq.QuadPart = -50000;
199
200 /* Sleep for this sample */
201 KeDelayExecutionThread(KernelMode,
202 FALSE,
203 &CurrentSample->PerfFreq);
204
205 /* Do another dummy CPUID */
206 KiCpuId(&CpuInfo, 0);
207
208 /* Fill out the ending data */
209 CurrentSample->PerfEnd =
210 KeQueryPerformanceCounter(&CurrentSample->PerfFreq);
211 CurrentSample->TSCEnd = __rdtsc();
212
213 /* Calculate the differences */
214 CurrentSample->PerfDelta = CurrentSample->PerfEnd.QuadPart -
215 CurrentSample->PerfStart.QuadPart;
216 CurrentSample->TSCDelta = CurrentSample->TSCEnd -
217 CurrentSample->TSCStart;
218
219 /* Compute CPU Speed */
220 CurrentSample->MHz = (ULONG)((CurrentSample->TSCDelta *
221 CurrentSample->
222 PerfFreq.QuadPart + 500000) /
223 (CurrentSample->PerfDelta *
224 1000000));
225
226 /* Check if this isn't the first sample */
227 if (Sample)
228 {
229 /* Check if we got a good precision within 1MHz */
230 if ((CurrentSample->MHz == CurrentSample[-1].MHz) ||
231 (CurrentSample->MHz == CurrentSample[-1].MHz + 1) ||
232 (CurrentSample->MHz == CurrentSample[-1].MHz - 1))
233 {
234 /* We did, stop sampling */
235 break;
236 }
237 }
238
239 /* Move on */
240 CurrentSample++;
241 Sample++;
242
243 if (Sample == sizeof(Samples) / sizeof(Samples[0]))
244 {
245 /* Restart */
246 CurrentSample = Samples;
247 Sample = 0;
248 }
249 }
250
251 /* Save the CPU Speed */
252 KeGetCurrentPrcb()->MHz = CurrentSample[-1].MHz;
253 }
254
255 /* Check if we have MTRR */
256 if (KeFeatureBits & KF_MTRR)
257 {
258 /* Then manually initialize MTRR for the CPU */
259 KiInitializeMTRR(i ? FALSE : TRUE);
260 }
261
262 /* Check if we have AMD MTRR and initialize it for the CPU */
263 if (KeFeatureBits & KF_AMDK6MTRR) KiAmdK6InitializeMTRR();
264
265 /* Check if this is a buggy Pentium and apply the fixup if so */
266 if (KiI386PentiumLockErrataPresent) KiI386PentiumLockErrataFixup();
267
268 /* Check if the CPU supports FXSR */
269 if (KeFeatureBits & KF_FXSR)
270 {
271 /* Get the current thread NPX state */
272 FxSaveArea = KiGetThreadNpxArea(KeGetCurrentThread());
273
274 /* Clear initial MXCsr mask */
275 FxSaveArea->U.FxArea.MXCsrMask = 0;
276
277 /* Save the current NPX State */
278 Ke386SaveFpuState(FxSaveArea);
279
280 /* Check if the current mask doesn't match the reserved bits */
281 if (FxSaveArea->U.FxArea.MXCsrMask != 0)
282 {
283 /* Then use whatever it's holding */
284 MXCsrMask = FxSaveArea->U.FxArea.MXCsrMask;
285 }
286
287 /* Check if nobody set the kernel-wide mask */
288 if (!KiMXCsrMask)
289 {
290 /* Then use the one we calculated above */
291 KiMXCsrMask = MXCsrMask;
292 }
293 else
294 {
295 /* Was it set to the same value we found now? */
296 if (KiMXCsrMask != MXCsrMask)
297 {
298 /* No, something is definitely wrong */
299 KeBugCheckEx(MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED,
300 KF_FXSR,
301 KiMXCsrMask,
302 MXCsrMask,
303 0);
304 }
305 }
306
307 /* Now set the kernel mask */
308 KiMXCsrMask &= MXCsrMask;
309 }
310 }
311 }
312
313 /* Return affinity back to where it was */
314 KeRevertToUserAffinityThread();
315
316 /* NT allows limiting the duration of an ISR with a registry key */
317 if (KiTimeLimitIsrMicroseconds)
318 {
319 /* FIXME: TODO */
320 DPRINT1("ISR Time Limit not yet supported\n");
321 }
322
323 /* Set CR0 features based on detected CPU */
324 KiSetCR0Bits();
325 }
326
327 INIT_SECTION
328 VOID
329 NTAPI
330 KiInitializePcr(IN ULONG ProcessorNumber,
331 IN PKIPCR Pcr,
332 IN PKIDTENTRY Idt,
333 IN PKGDTENTRY Gdt,
334 IN PKTSS Tss,
335 IN PKTHREAD IdleThread,
336 IN PVOID DpcStack)
337 {
338 /* Setup the TIB */
339 Pcr->NtTib.ExceptionList = EXCEPTION_CHAIN_END;
340 Pcr->NtTib.StackBase = 0;
341 Pcr->NtTib.StackLimit = 0;
342 Pcr->NtTib.Self = NULL;
343
344 /* Set the Current Thread */
345 Pcr->PrcbData.CurrentThread = IdleThread;
346
347 /* Set pointers to ourselves */
348 Pcr->SelfPcr = (PKPCR)Pcr;
349 Pcr->Prcb = &Pcr->PrcbData;
350
351 /* Set the PCR Version */
352 Pcr->MajorVersion = PCR_MAJOR_VERSION;
353 Pcr->MinorVersion = PCR_MINOR_VERSION;
354
355 /* Set the PCRB Version */
356 Pcr->PrcbData.MajorVersion = 1;
357 Pcr->PrcbData.MinorVersion = 1;
358
359 /* Set the Build Type */
360 Pcr->PrcbData.BuildType = 0;
361 #ifndef CONFIG_SMP
362 Pcr->PrcbData.BuildType |= PRCB_BUILD_UNIPROCESSOR;
363 #endif
364 #if DBG
365 Pcr->PrcbData.BuildType |= PRCB_BUILD_DEBUG;
366 #endif
367
368 /* Set the Processor Number and current Processor Mask */
369 Pcr->PrcbData.Number = (UCHAR)ProcessorNumber;
370 Pcr->PrcbData.SetMember = 1 << ProcessorNumber;
371
372 /* Set the PRCB for this Processor */
373 KiProcessorBlock[ProcessorNumber] = Pcr->Prcb;
374
375 /* Start us out at PASSIVE_LEVEL */
376 Pcr->Irql = PASSIVE_LEVEL;
377
378 /* Set the GDI, IDT, TSS and DPC Stack */
379 Pcr->GDT = (PVOID)Gdt;
380 Pcr->IDT = Idt;
381 Pcr->TSS = Tss;
382 Pcr->TssCopy = Tss;
383 Pcr->PrcbData.DpcStack = DpcStack;
384
385 /* Setup the processor set */
386 Pcr->PrcbData.MultiThreadProcessorSet = Pcr->PrcbData.SetMember;
387 }
388
389 INIT_SECTION
390 VOID
391 NTAPI
392 KiInitializeKernel(IN PKPROCESS InitProcess,
393 IN PKTHREAD InitThread,
394 IN PVOID IdleStack,
395 IN PKPRCB Prcb,
396 IN CCHAR Number,
397 IN PLOADER_PARAMETER_BLOCK LoaderBlock)
398 {
399 BOOLEAN NpxPresent;
400 ULONG FeatureBits;
401 ULONG PageDirectory[2];
402 PVOID DpcStack;
403 ULONG Vendor[3];
404 KIRQL DummyIrql;
405
406 /* Detect and set the CPU Type */
407 KiSetProcessorType();
408
409 /* Check if an FPU is present */
410 NpxPresent = KiIsNpxPresent();
411
412 /* Initialize the Power Management Support for this PRCB */
413 PoInitializePrcb(Prcb);
414
415 /* Bugcheck if this is a 386 CPU */
416 if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
417
418 /* Get the processor features for the CPU */
419 FeatureBits = KiGetFeatureBits();
420
421 /* Set the default NX policy (opt-in) */
422 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
423
424 /* Check if NPX is always on */
425 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
426 {
427 /* Set it always on */
428 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
429 FeatureBits |= KF_NX_ENABLED;
430 }
431 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
432 {
433 /* Set it in opt-out mode */
434 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
435 FeatureBits |= KF_NX_ENABLED;
436 }
437 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
438 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
439 {
440 /* Set the feature bits */
441 FeatureBits |= KF_NX_ENABLED;
442 }
443 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
444 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
445 {
446 /* Set disabled mode */
447 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
448 FeatureBits |= KF_NX_DISABLED;
449 }
450
451 /* Save feature bits */
452 Prcb->FeatureBits = FeatureBits;
453
454 /* Save CPU state */
455 KiSaveProcessorControlState(&Prcb->ProcessorState);
456
457 /* Get cache line information for this CPU */
458 KiGetCacheInformation();
459
460 /* Initialize spinlocks and DPC data */
461 KiInitSpinLocks(Prcb, Number);
462
463 /* Check if this is the Boot CPU */
464 if (!Number)
465 {
466 /* Set Node Data */
467 KeNodeBlock[0] = &KiNode0;
468 Prcb->ParentNode = KeNodeBlock[0];
469 KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
470
471 /* Set boot-level flags */
472 KeI386NpxPresent = NpxPresent;
473 KeI386CpuType = Prcb->CpuType;
474 KeI386CpuStep = Prcb->CpuStep;
475 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
476 KeProcessorLevel = (USHORT)Prcb->CpuType;
477 if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
478 KeFeatureBits = FeatureBits;
479 KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
480 KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
481
482 /* Detect 8-byte compare exchange support */
483 if (!(KeFeatureBits & KF_CMPXCHG8B))
484 {
485 /* Copy the vendor string */
486 RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
487
488 /* Bugcheck the system. Windows *requires* this */
489 KeBugCheckEx(UNSUPPORTED_PROCESSOR,
490 (1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
491 Vendor[0],
492 Vendor[1],
493 Vendor[2]);
494 }
495
496 /* Set the current MP Master KPRCB to the Boot PRCB */
497 Prcb->MultiThreadSetMaster = Prcb;
498
499 /* Lower to APC_LEVEL */
500 KeLowerIrql(APC_LEVEL);
501
502 /* Initialize some spinlocks */
503 KeInitializeSpinLock(&KiFreezeExecutionLock);
504 KeInitializeSpinLock(&Ki486CompatibilityLock);
505
506 /* Initialize portable parts of the OS */
507 KiInitSystem();
508
509 /* Initialize the Idle Process and the Process Listhead */
510 InitializeListHead(&KiProcessListHead);
511 PageDirectory[0] = 0;
512 PageDirectory[1] = 0;
513 KeInitializeProcess(InitProcess,
514 0,
515 0xFFFFFFFF,
516 PageDirectory,
517 FALSE);
518 InitProcess->QuantumReset = MAXCHAR;
519 }
520 else
521 {
522 /* FIXME */
523 DPRINT1("SMP Boot support not yet present\n");
524 }
525
526 /* Setup the Idle Thread */
527 KeInitializeThread(InitProcess,
528 InitThread,
529 NULL,
530 NULL,
531 NULL,
532 NULL,
533 NULL,
534 IdleStack);
535 InitThread->NextProcessor = Number;
536 InitThread->Priority = HIGH_PRIORITY;
537 InitThread->State = Running;
538 InitThread->Affinity = 1 << Number;
539 InitThread->WaitIrql = DISPATCH_LEVEL;
540 InitProcess->ActiveProcessors = 1 << Number;
541
542 /* HACK for MmUpdatePageDir */
543 ((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
544
545 /* Set basic CPU Features that user mode can read */
546 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
547 (KeFeatureBits & KF_MMX) ? TRUE: FALSE;
548 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
549 (KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
550 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
551 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
552 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
553 ((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
554 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
555 (KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
556 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
557 (KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
558
559 /* Set up the thread-related fields in the PRCB */
560 Prcb->CurrentThread = InitThread;
561 Prcb->NextThread = NULL;
562 Prcb->IdleThread = InitThread;
563
564 /* Initialize the Kernel Executive */
565 ExpInitializeExecutive(Number, LoaderBlock);
566
567 /* Only do this on the boot CPU */
568 if (!Number)
569 {
570 /* Calculate the time reciprocal */
571 KiTimeIncrementReciprocal =
572 KiComputeReciprocal(KeMaximumIncrement,
573 &KiTimeIncrementShiftCount);
574
575 /* Update DPC Values in case they got updated by the executive */
576 Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
577 Prcb->MinimumDpcRate = KiMinimumDpcRate;
578 Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
579
580 /* Allocate the DPC Stack */
581 DpcStack = MmCreateKernelStack(FALSE, 0);
582 if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
583 Prcb->DpcStack = DpcStack;
584
585 /* Allocate the IOPM save area. */
586 Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
587 PAGE_SIZE * 2,
588 ' eK');
589 if (!Ki386IopmSaveArea)
590 {
591 /* Bugcheck. We need this for V86/VDM support. */
592 KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
593 }
594 }
595
596 /* Raise to Dispatch */
597 KeRaiseIrql(DISPATCH_LEVEL,
598 &DummyIrql);
599
600 /* Set the Idle Priority to 0. This will jump into Phase 1 */
601 KeSetPriorityThread(InitThread, 0);
602
603 /* If there's no thread scheduled, put this CPU in the Idle summary */
604 KiAcquirePrcbLock(Prcb);
605 if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
606 KiReleasePrcbLock(Prcb);
607
608 /* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
609 KeRaiseIrql(HIGH_LEVEL,
610 &DummyIrql);
611 LoaderBlock->Prcb = 0;
612 }
613
614 INIT_SECTION
615 VOID
616 FASTCALL
617 KiGetMachineBootPointers(IN PKGDTENTRY *Gdt,
618 IN PKIDTENTRY *Idt,
619 IN PKIPCR *Pcr,
620 IN PKTSS *Tss)
621 {
622 KDESCRIPTOR GdtDescriptor, IdtDescriptor;
623 KGDTENTRY TssSelector, PcrSelector;
624 USHORT Tr, Fs;
625
626 /* Get GDT and IDT descriptors */
627 Ke386GetGlobalDescriptorTable(&GdtDescriptor.Limit);
628 __sidt(&IdtDescriptor.Limit);
629
630 /* Save IDT and GDT */
631 *Gdt = (PKGDTENTRY)GdtDescriptor.Base;
632 *Idt = (PKIDTENTRY)IdtDescriptor.Base;
633
634 /* Get TSS and FS Selectors */
635 Tr = Ke386GetTr();
636 Fs = Ke386GetFs();
637
638 /* Get PCR Selector, mask it and get its GDT Entry */
639 PcrSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Fs & ~RPL_MASK));
640
641 /* Get the KPCR itself */
642 *Pcr = (PKIPCR)(ULONG_PTR)(PcrSelector.BaseLow |
643 PcrSelector.HighWord.Bytes.BaseMid << 16 |
644 PcrSelector.HighWord.Bytes.BaseHi << 24);
645
646 /* Get TSS Selector, mask it and get its GDT Entry */
647 TssSelector = *(PKGDTENTRY)((ULONG_PTR)*Gdt + (Tr & ~RPL_MASK));
648
649 /* Get the KTSS itself */
650 *Tss = (PKTSS)(ULONG_PTR)(TssSelector.BaseLow |
651 TssSelector.HighWord.Bytes.BaseMid << 16 |
652 TssSelector.HighWord.Bytes.BaseHi << 24);
653 }
654
655 INIT_SECTION
656 VOID
657 NTAPI
658 KiSystemStartupBootStack(VOID)
659 {
660 PKTHREAD Thread;
661
662 /* Initialize the kernel for the current CPU */
663 KiInitializeKernel(&KiInitialProcess.Pcb,
664 (PKTHREAD)KeLoaderBlock->Thread,
665 (PVOID)(KeLoaderBlock->KernelStack & ~3),
666 (PKPRCB)__readfsdword(KPCR_PRCB),
667 KeNumberProcessors - 1,
668 KeLoaderBlock);
669
670 /* Set the priority of this thread to 0 */
671 Thread = KeGetCurrentThread();
672 Thread->Priority = 0;
673
674 /* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
675 _enable();
676 KeLowerIrql(DISPATCH_LEVEL);
677
678 /* Set the right wait IRQL */
679 Thread->WaitIrql = DISPATCH_LEVEL;
680
681 /* Jump into the idle loop */
682 KiIdleLoop();
683 }
684
685 static
686 VOID
687 KiMarkPageAsReadOnly(
688 PVOID Address)
689 {
690 PHARDWARE_PTE PointerPte;
691
692 /* Make sure the address is page aligned */
693 ASSERT(ALIGN_DOWN_POINTER_BY(Address, PAGE_SIZE) == Address);
694
695 /* Get the PTE address */
696 PointerPte = ((PHARDWARE_PTE)PTE_BASE) + ((ULONG_PTR)Address / PAGE_SIZE);
697 ASSERT(PointerPte->Valid);
698 ASSERT(PointerPte->Write);
699
700 /* Set as read-only */
701 PointerPte->Write = 0;
702
703 /* Flush the TLB entry */
704 __invlpg(Address);
705 }
706
707 INIT_SECTION
708 VOID
709 NTAPI
710 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
711 {
712 ULONG Cpu;
713 PKTHREAD InitialThread;
714 ULONG InitialStack;
715 PKGDTENTRY Gdt;
716 PKIDTENTRY Idt;
717 KIDTENTRY NmiEntry, DoubleFaultEntry;
718 PKTSS Tss;
719 PKIPCR Pcr;
720 KIRQL DummyIrql;
721
722 /* Boot cycles timestamp */
723 BootCycles = __rdtsc();
724
725 /* Save the loader block and get the current CPU */
726 KeLoaderBlock = LoaderBlock;
727 Cpu = KeNumberProcessors;
728 if (!Cpu)
729 {
730 /* If this is the boot CPU, set FS and the CPU Number*/
731 Ke386SetFs(KGDT_R0_PCR);
732 __writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
733
734 /* Set the initial stack and idle thread as well */
735 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
736 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
737 }
738
739 /* Save the initial thread and stack */
740 InitialStack = LoaderBlock->KernelStack;
741 InitialThread = (PKTHREAD)LoaderBlock->Thread;
742
743 /* Clean the APC List Head */
744 InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
745
746 /* Initialize the machine type */
747 KiInitializeMachineType();
748
749 /* Skip initial setup if this isn't the Boot CPU */
750 if (Cpu) goto AppCpuInit;
751
752 /* Get GDT, IDT, PCR and TSS pointers */
753 KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
754
755 /* Setup the TSS descriptors and entries */
756 Ki386InitializeTss(Tss, Idt, Gdt);
757
758 /* Initialize the PCR */
759 RtlZeroMemory(Pcr, PAGE_SIZE);
760 KiInitializePcr(Cpu,
761 Pcr,
762 Idt,
763 Gdt,
764 Tss,
765 InitialThread,
766 (PVOID)KiDoubleFaultStack);
767
768 /* Set us as the current process */
769 InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
770
771 /* Clear DR6/7 to cleanup bootloader debugging */
772 __writefsdword(KPCR_TEB, 0);
773 __writefsdword(KPCR_DR6, 0);
774 __writefsdword(KPCR_DR7, 0);
775
776 /* Setup the IDT */
777 KeInitExceptions();
778
779 /* Load Ring 3 selectors for DS/ES */
780 Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
781 Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
782
783 /* Save NMI and double fault traps */
784 RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
785 RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
786
787 /* Copy kernel's trap handlers */
788 RtlCopyMemory(Idt,
789 (PVOID)KiIdtDescriptor.Base,
790 KiIdtDescriptor.Limit + 1);
791
792 /* Restore NMI and double fault */
793 RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
794 RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
795
796 AppCpuInit:
797 /* Loop until we can release the freeze lock */
798 do
799 {
800 /* Loop until execution can continue */
801 while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
802 } while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
803
804 /* Setup CPU-related fields */
805 __writefsdword(KPCR_NUMBER, Cpu);
806 __writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
807 __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
808 __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
809
810 /* Initialize the Processor with HAL */
811 HalInitializeProcessor(Cpu, KeLoaderBlock);
812
813 /* Set active processors */
814 KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
815 KeNumberProcessors++;
816
817 /* Check if this is the boot CPU */
818 if (!Cpu)
819 {
820 /* Initialize debugging system */
821 KdInitSystem(0, KeLoaderBlock);
822
823 /* Check for break-in */
824 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
825
826 /* Make the lowest page of the boot and double fault stack read-only */
827 KiMarkPageAsReadOnly(P0BootStackData);
828 KiMarkPageAsReadOnly(KiDoubleFaultStackData);
829 }
830
831 /* Raise to HIGH_LEVEL */
832 KeRaiseIrql(HIGH_LEVEL,
833 &DummyIrql);
834
835 /* Switch to new kernel stack and start kernel bootstrapping */
836 KiSwitchToBootStack(InitialStack & ~3);
837 }