2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/amd64/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Timo Kreuzer (timo.kreuzer@reactos.org)
10 /* INCLUDES *****************************************************************/
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17 KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18 KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
20 /* GLOBALS *******************************************************************/
22 /* Function pointer for early debug prints */
23 ULONG (*FrLdrDbgPrint
)(const char *Format
, ...);
25 /* Spinlocks used only on X86 */
26 KSPIN_LOCK KiFreezeExecutionLock
;
31 /* Boot and double-fault/NMI/DPC stack */
32 UCHAR
DECLSPEC_ALIGN(16) P0BootStackData
[KERNEL_STACK_SIZE
] = {0};
33 UCHAR
DECLSPEC_ALIGN(16) KiDoubleFaultStackData
[KERNEL_STACK_SIZE
] = {0};
34 ULONG_PTR P0BootStack
= (ULONG_PTR
)&P0BootStackData
[KERNEL_STACK_SIZE
];
35 ULONG_PTR KiDoubleFaultStack
= (ULONG_PTR
)&KiDoubleFaultStackData
[KERNEL_STACK_SIZE
];
37 void KiInitializeSegments();
38 void KiSystemCallEntry64();
39 void KiSystemCallEntry32();
41 /* FUNCTIONS *****************************************************************/
46 KiInitMachineDependent(VOID
)
48 /* Check for large page support */
49 if (KeFeatureBits
& KF_LARGE_PAGE
)
51 /* FIXME: Support this */
52 DPRINT("Large Page support detected but not yet taken advantage of!\n");
55 /* Check for global page support */
56 if (KeFeatureBits
& KF_GLOBAL_PAGE
)
58 /* FIXME: Support this */
59 DPRINT("Global Page support detected but not yet taken advantage of!\n");
62 /* Check if we have MTRR */
63 if (KeFeatureBits
& KF_MTRR
)
65 /* FIXME: Support this */
66 DPRINT("MTRR support detected but not yet taken advantage of!\n");
69 /* Check for PAT and/or MTRR support */
70 if (KeFeatureBits
& KF_PAT
)
72 /* FIXME: Support this */
73 DPRINT("PAT support detected but not yet taken advantage of!\n");
76 /* Allocate the IOPM save area. */
77 // Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
79 // TAG('K', 'e', ' ', ' '));
80 // if (!Ki386IopmSaveArea)
82 // /* Bugcheck. We need this for V86/VDM support. */
83 // KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
90 KiInitializePcr(IN PKIPCR Pcr
,
91 IN ULONG ProcessorNumber
,
92 IN PKTHREAD IdleThread
,
95 KDESCRIPTOR GdtDescriptor
= {{0},0,0}, IdtDescriptor
= {{0},0,0};
96 PKGDTENTRY64 TssEntry
;
99 /* Zero out the PCR */
100 RtlZeroMemory(Pcr
, sizeof(KIPCR
));
102 /* Set pointers to ourselves */
103 Pcr
->Self
= (PKPCR
)Pcr
;
104 Pcr
->CurrentPrcb
= &Pcr
->Prcb
;
106 /* Set the PCR Version */
107 Pcr
->MajorVersion
= PCR_MAJOR_VERSION
;
108 Pcr
->MinorVersion
= PCR_MINOR_VERSION
;
110 /* Set the PRCB Version */
111 Pcr
->Prcb
.MajorVersion
= 1;
112 Pcr
->Prcb
.MinorVersion
= 1;
114 /* Set the Build Type */
115 Pcr
->Prcb
.BuildType
= 0;
117 Pcr
->Prcb
.BuildType
|= PRCB_BUILD_UNIPROCESSOR
;
120 Pcr
->Prcb
.BuildType
|= PRCB_BUILD_DEBUG
;
123 /* Set the Processor Number and current Processor Mask */
124 Pcr
->Prcb
.Number
= (UCHAR
)ProcessorNumber
;
125 Pcr
->Prcb
.SetMember
= 1ULL << ProcessorNumber
;
127 /* Get GDT and IDT descriptors */
128 __sgdt(&GdtDescriptor
.Limit
);
129 __sidt(&IdtDescriptor
.Limit
);
130 Pcr
->GdtBase
= (PVOID
)GdtDescriptor
.Base
;
131 Pcr
->IdtBase
= (PKIDTENTRY
)IdtDescriptor
.Base
;
133 /* Get TSS Selector */
135 ASSERT(Tr
== KGDT64_SYS_TSS
);
138 TssEntry
= KiGetGdtEntry(Pcr
->GdtBase
, Tr
);
140 /* Get the KTSS itself */
141 Pcr
->TssBase
= KiGetGdtDescriptorBase(TssEntry
);
143 Pcr
->Prcb
.RspBase
= Pcr
->TssBase
->Rsp0
; // FIXME
146 Pcr
->Prcb
.DpcStack
= DpcStack
;
148 /* Setup the processor set */
149 Pcr
->Prcb
.MultiThreadProcessorSet
= Pcr
->Prcb
.SetMember
;
151 /* Clear DR6/7 to cleanup bootloader debugging */
152 Pcr
->Prcb
.ProcessorState
.SpecialRegisters
.KernelDr6
= 0;
153 Pcr
->Prcb
.ProcessorState
.SpecialRegisters
.KernelDr7
= 0;
155 /* Set the Current Thread */
156 Pcr
->Prcb
.CurrentThread
= IdleThread
;
158 /* Start us out at PASSIVE_LEVEL */
159 Pcr
->Irql
= PASSIVE_LEVEL
;
160 KeSetCurrentIrql(PASSIVE_LEVEL
);
165 KiInitializeCpu(PKIPCR Pcr
)
171 KiInitializeSegments();
174 __writemsr(MSR_GS_BASE
, (ULONG64
)Pcr
);
175 __writemsr(MSR_GS_SWAP
, (ULONG64
)Pcr
);
177 /* Detect and set the CPU Type */
178 KiSetProcessorType();
180 /* Get the processor features for this CPU */
181 FeatureBits
= KiGetFeatureBits();
183 /* Check if we support all needed features */
184 if ((FeatureBits
& REQUIRED_FEATURE_BITS
) != REQUIRED_FEATURE_BITS
)
186 /* If not, bugcheck system */
187 FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
188 FeatureBits
, REQUIRED_FEATURE_BITS
);
192 /* Set DEP to always on */
193 SharedUserData
->NXSupportPolicy
= NX_SUPPORT_POLICY_ALWAYSON
;
194 FeatureBits
|= KF_NX_ENABLED
;
196 /* Save feature bits */
197 Pcr
->Prcb
.FeatureBits
= FeatureBits
;
199 /* Enable fx save restore support */
200 __writecr4(__readcr4() | CR4_FXSR
);
202 /* Enable XMMI exceptions */
203 __writecr4(__readcr4() | CR4_XMMEXCPT
);
205 /* Enable Write-Protection */
206 __writecr0(__readcr0() | CR0_WP
);
208 /* Disable fpu monitoring */
209 __writecr0(__readcr0() & ~CR0_MP
);
211 /* Disable x87 fpu exceptions */
212 __writecr0(__readcr0() & ~CR0_NE
);
217 /* Set the systemcall entry points */
218 __writemsr(MSR_LSTAR
, (ULONG64
)KiSystemCallEntry64
);
219 __writemsr(MSR_CSTAR
, (ULONG64
)KiSystemCallEntry32
);
221 __writemsr(MSR_STAR
, ((ULONG64
)KGDT64_R0_CODE
<< 32) |
222 ((ULONG64
)(KGDT64_R3_CMCODE
|RPL_MASK
) << 48));
224 /* Set the flags to be cleared when doing a syscall */
225 __writemsr(MSR_SYSCALL_MASK
, EFLAGS_IF_MASK
| EFLAGS_TF
| EFLAGS_DF
);
227 /* Enable syscall instruction and no-execute support */
228 __writemsr(MSR_EFER
, __readmsr(MSR_EFER
) | MSR_SCE
| MSR_NXE
);
230 /* Initialize the PAT */
231 Pat
= (PAT_WB
<< 0) | (PAT_WC
<< 8) | (PAT_UCM
<< 16) | (PAT_UC
<< 24) |
232 (PAT_WB
<< 32) | (PAT_WC
<< 40) | (PAT_UCM
<< 48) | (PAT_UC
<< 56);
233 __writemsr(MSR_PAT
, Pat
);
238 KiInitializeTss(IN PKTSS64 Tss
,
241 PKGDTENTRY64 TssEntry
;
243 /* Get pointer to the GDT entry */
244 TssEntry
= KiGetGdtEntry(KeGetPcr()->GdtBase
, KGDT64_SYS_TSS
);
246 /* Initialize the GDT entry */
247 KiInitGdtEntry(TssEntry
, (ULONG64
)Tss
, sizeof(KTSS64
), AMD64_TSS
, 0);
249 /* Zero out the TSS */
250 RtlZeroMemory(Tss
, sizeof(KTSS64
));
252 /* FIXME: I/O Map? */
253 Tss
->IoMapBase
= 0x68;
255 /* Setup ring 0 stack pointer */
258 /* Setup a stack for Double Fault Traps */
259 Tss
->Ist
[1] = (ULONG64
)KiDoubleFaultStack
;
261 /* Setup a stack for CheckAbort Traps */
262 Tss
->Ist
[2] = (ULONG64
)KiDoubleFaultStack
;
264 /* Setup a stack for NMI Traps */
265 Tss
->Ist
[3] = (ULONG64
)KiDoubleFaultStack
;
267 /* Load the task register */
268 __ltr(KGDT64_SYS_TSS
);
274 KiInitializeKernelMachineDependent(
276 IN PLOADER_PARAMETER_BLOCK LoaderBlock
)
278 /* Set boot-level flags */
279 KeI386CpuType
= Prcb
->CpuType
;
280 KeI386CpuStep
= Prcb
->CpuStep
;
281 KeProcessorArchitecture
= PROCESSOR_ARCHITECTURE_AMD64
;
282 KeProcessorLevel
= (USHORT
)Prcb
->CpuType
;
284 KeProcessorRevision
= Prcb
->CpuStep
;
286 /* Set basic CPU Features that user mode can read */
287 SharedUserData
->ProcessorFeatures
[PF_COMPARE_EXCHANGE_DOUBLE
] = TRUE
;
288 SharedUserData
->ProcessorFeatures
[PF_RDTSC_INSTRUCTION_AVAILABLE
] = TRUE
;
289 SharedUserData
->ProcessorFeatures
[PF_PPC_MOVEMEM_64BIT_OK
] = TRUE
;
290 SharedUserData
->ProcessorFeatures
[PF_PAE_ENABLED
] = TRUE
; // ???
291 SharedUserData
->ProcessorFeatures
[PF_NX_ENABLED
] = TRUE
;
292 SharedUserData
->ProcessorFeatures
[PF_FASTFAIL_AVAILABLE
] = TRUE
;
293 SharedUserData
->ProcessorFeatures
[PF_XSAVE_ENABLED
] = TRUE
;
294 SharedUserData
->ProcessorFeatures
[PF_MMX_INSTRUCTIONS_AVAILABLE
] =
295 (Prcb
->FeatureBits
& KF_MMX
) ? TRUE
: FALSE
;
296 SharedUserData
->ProcessorFeatures
[PF_XMMI_INSTRUCTIONS_AVAILABLE
] =
297 ((Prcb
->FeatureBits
& KF_FXSR
) && (Prcb
->FeatureBits
& KF_XMMI
)) ? TRUE
: FALSE
;
298 SharedUserData
->ProcessorFeatures
[PF_XMMI64_INSTRUCTIONS_AVAILABLE
] =
299 ((Prcb
->FeatureBits
& KF_FXSR
) && (Prcb
->FeatureBits
& KF_XMMI64
)) ? TRUE
: FALSE
;
300 SharedUserData
->ProcessorFeatures
[PF_3DNOW_INSTRUCTIONS_AVAILABLE
] =
301 (Prcb
->FeatureBits
& KF_3DNOW
) ? TRUE
: FALSE
;
302 SharedUserData
->ProcessorFeatures
[PF_SSE3_INSTRUCTIONS_AVAILABLE
] =
303 (Prcb
->FeatureBits
& KF_SSE3
) ? TRUE
: FALSE
;
304 SharedUserData
->ProcessorFeatures
[PF_COMPARE_EXCHANGE128
] =
305 (Prcb
->FeatureBits
& KF_CMPXCHG16B
) ? TRUE
: FALSE
;
307 /* Set the default NX policy (opt-in) */
308 SharedUserData
->NXSupportPolicy
= NX_SUPPORT_POLICY_OPTIN
;
310 /* Check if NPX is always on */
311 if (strstr(KeLoaderBlock
->LoadOptions
, "NOEXECUTE=ALWAYSON"))
313 /* Set it always on */
314 SharedUserData
->NXSupportPolicy
= NX_SUPPORT_POLICY_ALWAYSON
;
315 Prcb
->FeatureBits
|= KF_NX_ENABLED
;
317 else if (strstr(KeLoaderBlock
->LoadOptions
, "NOEXECUTE=OPTOUT"))
319 /* Set it in opt-out mode */
320 SharedUserData
->NXSupportPolicy
= NX_SUPPORT_POLICY_OPTOUT
;
321 Prcb
->FeatureBits
|= KF_NX_ENABLED
;
323 else if ((strstr(KeLoaderBlock
->LoadOptions
, "NOEXECUTE=OPTIN")) ||
324 (strstr(KeLoaderBlock
->LoadOptions
, "NOEXECUTE")))
326 /* Set the feature bits */
327 Prcb
->FeatureBits
|= KF_NX_ENABLED
;
329 else if ((strstr(KeLoaderBlock
->LoadOptions
, "NOEXECUTE=ALWAYSOFF")) ||
330 (strstr(KeLoaderBlock
->LoadOptions
, "EXECUTE")))
332 /* Set disabled mode */
333 SharedUserData
->NXSupportPolicy
= NX_SUPPORT_POLICY_ALWAYSOFF
;
334 Prcb
->FeatureBits
|= KF_NX_DISABLED
;
338 static LDR_DATA_TABLE_ENTRY LdrCoreEntries
[3];
341 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock
)
343 PLDR_DATA_TABLE_ENTRY LdrEntry
;
347 /* Initialize the list head */
348 InitializeListHead(&PsLoadedModuleList
);
350 /* Loop the first 3 entries */
351 for (Entry
= LoaderBlock
->LoadOrderListHead
.Flink
, i
= 0;
352 Entry
!= &LoaderBlock
->LoadOrderListHead
&& i
< 3;
353 Entry
= Entry
->Flink
, i
++)
355 /* Get the data table entry */
356 LdrEntry
= CONTAINING_RECORD(Entry
,
357 LDR_DATA_TABLE_ENTRY
,
361 LdrCoreEntries
[i
] = *LdrEntry
;
363 /* Insert the copy into the list */
364 InsertTailList(&PsLoadedModuleList
, &LdrCoreEntries
[i
].InLoadOrderLinks
);
371 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock
)
374 PKTHREAD InitialThread
;
375 ULONG64 InitialStack
;
379 FrLdrDbgPrint
= LoaderBlock
->u
.I386
.CommonDataArea
;
380 //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
382 /* Save the loader block */
383 KeLoaderBlock
= LoaderBlock
;
385 /* Get the current CPU number */
386 Cpu
= KeNumberProcessors
++; // FIXME
388 /* LoaderBlock initialization for Cpu 0 */
391 /* Set the initial stack, idle thread and process */
392 LoaderBlock
->KernelStack
= (ULONG_PTR
)P0BootStack
;
393 LoaderBlock
->Thread
= (ULONG_PTR
)&KiInitialThread
;
394 LoaderBlock
->Process
= (ULONG_PTR
)&KiInitialProcess
.Pcb
;
395 LoaderBlock
->Prcb
= (ULONG_PTR
)&KiInitialPcr
.Prcb
;
398 /* Get Pcr from loader block */
399 Pcr
= CONTAINING_RECORD(LoaderBlock
->Prcb
, KIPCR
, Prcb
);
401 /* Set the PRCB for this Processor */
402 KiProcessorBlock
[Cpu
] = &Pcr
->Prcb
;
404 /* Align stack to 16 bytes */
405 LoaderBlock
->KernelStack
&= ~(16 - 1);
407 /* Save the initial thread and stack */
408 InitialStack
= LoaderBlock
->KernelStack
; // Checkme
409 InitialThread
= (PKTHREAD
)LoaderBlock
->Thread
;
411 /* Set us as the current process */
412 InitialThread
->ApcState
.Process
= (PVOID
)LoaderBlock
->Process
;
414 /* Initialize the PCR */
415 KiInitializePcr(Pcr
, Cpu
, InitialThread
, (PVOID
)KiDoubleFaultStack
);
417 /* Initialize the CPU features */
418 KiInitializeCpu(Pcr
);
420 /* Initial setup for the boot CPU */
423 /* Initialize the module list (ntos, hal, kdcom) */
424 KiInitModuleList(LoaderBlock
);
426 /* Setup the TSS descriptors and entries */
427 KiInitializeTss(Pcr
->TssBase
, InitialStack
);
432 /* Initialize debugging system */
433 KdInitSystem(0, KeLoaderBlock
);
435 /* Check for break-in */
436 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C
);
439 DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
440 Pcr
, Pcr
->GdtBase
, Pcr
->IdtBase
, Pcr
->TssBase
);
443 while (InterlockedBitTestAndSet64((PLONG64
)&KiFreezeExecutionLock
, 0))
445 /* Loop until lock is free */
446 while ((*(volatile KSPIN_LOCK
*)&KiFreezeExecutionLock
) & 1);
449 /* Initialize the Processor with HAL */
450 HalInitializeProcessor(Cpu
, KeLoaderBlock
);
452 /* Set processor as active */
453 KeActiveProcessors
|= 1ULL << Cpu
;
456 InterlockedAnd64((PLONG64
)&KiFreezeExecutionLock
, 0);
458 /* Raise to HIGH_LEVEL */
459 KfRaiseIrql(HIGH_LEVEL
);
461 /* Machine specific kernel initialization */
462 if (Cpu
== 0) KiInitializeKernelMachineDependent(&Pcr
->Prcb
, LoaderBlock
);
464 /* Switch to new kernel stack and start kernel bootstrapping */
465 KiSwitchToBootStack(InitialStack
& ~3);