[RTL]
[reactos.git] / reactos / ntoskrnl / ke / amd64 / kiinit.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Timo Kreuzer (timo.kreuzer@reactos.org)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17 KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18 KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT)
19
20 /* GLOBALS *******************************************************************/
21
22 /* Function pointer for early debug prints */
23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
24
25 /* Spinlocks used only on X86 */
26 KSPIN_LOCK KiFreezeExecutionLock;
27
28
29 KIPCR KiInitialPcr;
30
31 /* Boot and double-fault/NMI/DPC stack */
32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
36
37 void KiInitializeSegments();
38 void KiSystemCallEntry64();
39 void KiSystemCallEntry32();
40
41 /* FUNCTIONS *****************************************************************/
42
43 VOID
44 NTAPI
45 KiInitMachineDependent(VOID)
46 {
47 /* Check for large page support */
48 if (KeFeatureBits & KF_LARGE_PAGE)
49 {
50 /* FIXME: Support this */
51 DPRINT("Large Page support detected but not yet taken advantage of!\n");
52 }
53
54 /* Check for global page support */
55 if (KeFeatureBits & KF_GLOBAL_PAGE)
56 {
57 /* FIXME: Support this */
58 DPRINT("Global Page support detected but not yet taken advantage of!\n");
59 }
60
61 /* Check if we have MTRR */
62 if (KeFeatureBits & KF_MTRR)
63 {
64 /* FIXME: Support this */
65 DPRINT("MTRR support detected but not yet taken advantage of!\n");
66 }
67
68 /* Check for PAT and/or MTRR support */
69 if (KeFeatureBits & KF_PAT)
70 {
71 /* FIXME: Support this */
72 DPRINT("PAT support detected but not yet taken advantage of!\n");
73 }
74
75 /* Allocate the IOPM save area. */
76 // Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
77 // PAGE_SIZE * 2,
78 // TAG('K', 'e', ' ', ' '));
79 // if (!Ki386IopmSaveArea)
80 // {
81 // /* Bugcheck. We need this for V86/VDM support. */
82 // KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
83 // }
84
85 }
86
87 VOID
88 NTAPI
89 KiInitializePcr(IN PKIPCR Pcr,
90 IN ULONG ProcessorNumber,
91 IN PKTHREAD IdleThread,
92 IN PVOID DpcStack)
93 {
94 KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
95 PKGDTENTRY64 TssEntry;
96 USHORT Tr = 0;
97
98 /* Zero out the PCR */
99 RtlZeroMemory(Pcr, sizeof(KIPCR));
100
101 /* Set pointers to ourselves */
102 Pcr->Self = (PKPCR)Pcr;
103 Pcr->CurrentPrcb = &Pcr->Prcb;
104
105 /* Set the PCR Version */
106 Pcr->MajorVersion = PCR_MAJOR_VERSION;
107 Pcr->MinorVersion = PCR_MINOR_VERSION;
108
109 /* Set the PRCB Version */
110 Pcr->Prcb.MajorVersion = 1;
111 Pcr->Prcb.MinorVersion = 1;
112
113 /* Set the Build Type */
114 Pcr->Prcb.BuildType = 0;
115 #ifndef CONFIG_SMP
116 Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
117 #endif
118 #if DBG
119 Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
120 #endif
121
122 /* Set the Processor Number and current Processor Mask */
123 Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
124 Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
125
126 /* Get GDT and IDT descriptors */
127 __sgdt(&GdtDescriptor.Limit);
128 __sidt(&IdtDescriptor.Limit);
129 Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
130 Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
131
132 /* Get TSS Selector */
133 __str(&Tr);
134 ASSERT(Tr == KGDT64_SYS_TSS);
135
136 /* Get TSS Entry */
137 TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
138
139 /* Get the KTSS itself */
140 Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
141
142 Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
143
144 /* Set DPC Stack */
145 Pcr->Prcb.DpcStack = DpcStack;
146
147 /* Setup the processor set */
148 Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
149
150 /* Clear DR6/7 to cleanup bootloader debugging */
151 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
152 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
153
154 /* Set the Current Thread */
155 Pcr->Prcb.CurrentThread = IdleThread;
156
157 /* Start us out at PASSIVE_LEVEL */
158 Pcr->Irql = PASSIVE_LEVEL;
159 KeSetCurrentIrql(PASSIVE_LEVEL);
160 }
161
162 VOID
163 NTAPI
164 KiInitializeCpu(PKIPCR Pcr)
165 {
166 ULONG64 Pat;
167 ULONG FeatureBits;
168
169 /* Initialize gs */
170 KiInitializeSegments();
171
172 /* Set GS base */
173 __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
174 __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
175
176 /* Detect and set the CPU Type */
177 KiSetProcessorType();
178
179 /* Get the processor features for this CPU */
180 FeatureBits = KiGetFeatureBits();
181
182 /* Check if we support all needed features */
183 if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
184 {
185 /* If not, bugcheck system */
186 FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
187 FeatureBits, REQUIRED_FEATURE_BITS);
188 KeBugCheck(0);
189 }
190
191 /* Set DEP to always on */
192 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
193 FeatureBits |= KF_NX_ENABLED;
194
195 /* Save feature bits */
196 Pcr->Prcb.FeatureBits = FeatureBits;
197
198 /* Enable fx save restore support */
199 __writecr4(__readcr4() | CR4_FXSR);
200
201 /* Enable XMMI exceptions */
202 __writecr4(__readcr4() | CR4_XMMEXCPT);
203
204 /* Enable Write-Protection */
205 __writecr0(__readcr0() | CR0_WP);
206
207 /* Disable fpu monitoring */
208 __writecr0(__readcr0() & ~CR0_MP);
209
210 /* Disable x87 fpu exceptions */
211 __writecr0(__readcr0() & ~CR0_NE);
212
213 /* LDT is unused */
214 __lldt(0);
215
216 /* Set the systemcall entry points */
217 __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
218 __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
219
220 __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
221 ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
222
223 /* Set the flags to be cleared when doing a syscall */
224 __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
225
226 /* Enable syscall instruction and no-execute support */
227 __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
228
229 /* Initialize the PAT */
230 Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
231 (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
232 __writemsr(MSR_PAT, Pat);
233 }
234
235 VOID
236 FASTCALL
237 KiInitializeTss(IN PKTSS64 Tss,
238 IN UINT64 Stack)
239 {
240 PKGDTENTRY64 TssEntry;
241
242 /* Get pointer to the GDT entry */
243 TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
244
245 /* Initialize the GDT entry */
246 KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
247
248 /* Zero out the TSS */
249 RtlZeroMemory(Tss, sizeof(KTSS64));
250
251 /* FIXME: I/O Map? */
252 Tss->IoMapBase = 0x68;
253
254 /* Setup ring 0 stack pointer */
255 Tss->Rsp0 = Stack;
256
257 /* Setup a stack for Double Fault Traps */
258 Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
259
260 /* Setup a stack for CheckAbort Traps */
261 Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
262
263 /* Setup a stack for NMI Traps */
264 Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
265
266 /* Load the task register */
267 __ltr(KGDT64_SYS_TSS);
268 }
269
270 VOID
271 NTAPI
272 INIT_FUNCTION
273 KiInitializeKernelMachineDependent(
274 IN PKPRCB Prcb,
275 IN PLOADER_PARAMETER_BLOCK LoaderBlock)
276 {
277 /* Set boot-level flags */
278 KeI386CpuType = Prcb->CpuType;
279 KeI386CpuStep = Prcb->CpuStep;
280 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
281 KeProcessorLevel = (USHORT)Prcb->CpuType;
282 if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
283
284 /* Set basic CPU Features that user mode can read */
285 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
286 (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
287 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
288 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
289 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
290 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
291 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
292 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
293 (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
294 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
295
296 /* Set the default NX policy (opt-in) */
297 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
298
299 /* Check if NPX is always on */
300 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
301 {
302 /* Set it always on */
303 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
304 Prcb->FeatureBits |= KF_NX_ENABLED;
305 }
306 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
307 {
308 /* Set it in opt-out mode */
309 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
310 Prcb->FeatureBits |= KF_NX_ENABLED;
311 }
312 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
313 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
314 {
315 /* Set the feature bits */
316 Prcb->FeatureBits |= KF_NX_ENABLED;
317 }
318 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
319 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
320 {
321 /* Set disabled mode */
322 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
323 Prcb->FeatureBits |= KF_NX_DISABLED;
324 }
325
326 }
327
328 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
329
330 void
331 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
332 {
333 PLDR_DATA_TABLE_ENTRY LdrEntry;
334 PLIST_ENTRY Entry;
335 ULONG i;
336
337 /* Initialize the list head */
338 InitializeListHead(&PsLoadedModuleList);
339
340 /* Loop the first 3 entries */
341 for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
342 Entry != &LoaderBlock->LoadOrderListHead && i < 3;
343 Entry = Entry->Flink, i++)
344 {
345 /* Get the data table entry */
346 LdrEntry = CONTAINING_RECORD(Entry,
347 LDR_DATA_TABLE_ENTRY,
348 InLoadOrderLinks);
349
350 /* Copy the entry */
351 LdrCoreEntries[i] = *LdrEntry;
352
353 /* Insert the copy into the list */
354 InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
355 }
356 }
357
358 VOID
359 NTAPI
360 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
361 {
362 CCHAR Cpu;
363 PKTHREAD InitialThread;
364 ULONG64 InitialStack;
365 PKIPCR Pcr;
366
367 /* HACK */
368 FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
369 //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
370
371 /* Save the loader block */
372 KeLoaderBlock = LoaderBlock;
373
374 /* Get the current CPU number */
375 Cpu = (CCHAR)KeNumberProcessors++; // FIXME
376
377 /* LoaderBlock initialization for Cpu 0 */
378 if (Cpu == 0)
379 {
380 /* Set the initial stack, idle thread and process */
381 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
382 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
383 LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
384 LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
385 }
386
387 /* Get Pcr from loader block */
388 Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
389
390 /* Set the PRCB for this Processor */
391 KiProcessorBlock[Cpu] = &Pcr->Prcb;
392
393 /* Align stack to 16 bytes */
394 LoaderBlock->KernelStack &= ~(16 - 1);
395
396 /* Save the initial thread and stack */
397 InitialStack = LoaderBlock->KernelStack; // Checkme
398 InitialThread = (PKTHREAD)LoaderBlock->Thread;
399
400 /* Set us as the current process */
401 InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
402
403 /* Initialize the PCR */
404 KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
405
406 /* Initialize the CPU features */
407 KiInitializeCpu(Pcr);
408
409 /* Initial setup for the boot CPU */
410 if (Cpu == 0)
411 {
412 /* Initialize the module list (ntos, hal, kdcom) */
413 KiInitModuleList(LoaderBlock);
414
415 /* Setup the TSS descriptors and entries */
416 KiInitializeTss(Pcr->TssBase, InitialStack);
417
418 /* Setup the IDT */
419 KeInitExceptions();
420
421 /* Initialize debugging system */
422 KdInitSystem(0, KeLoaderBlock);
423
424 /* Check for break-in */
425 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
426 }
427
428 DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
429 Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
430
431 /* Acquire lock */
432 while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
433 {
434 /* Loop until lock is free */
435 while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
436 }
437
438 /* Initialize the Processor with HAL */
439 HalInitializeProcessor(Cpu, KeLoaderBlock);
440
441 /* Set processor as active */
442 KeActiveProcessors |= 1ULL << Cpu;
443
444 /* Release lock */
445 InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
446
447 /* Raise to HIGH_LEVEL */
448 KfRaiseIrql(HIGH_LEVEL);
449
450 /* Machine specific kernel initialization */
451 if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
452
453 /* Switch to new kernel stack and start kernel bootstrapping */
454 KiSwitchToBootStack(InitialStack & ~3);
455 }
456