[STORPORT] Fix x64 build
[reactos.git] / ntoskrnl / ke / amd64 / kiinit.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/amd64/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Timo Kreuzer (timo.kreuzer@reactos.org)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17 KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18 KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
19
20 /* GLOBALS *******************************************************************/
21
22 extern BOOLEAN RtlpUse16ByteSLists;
23
24 /* Function pointer for early debug prints */
25 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
26
27 /* Spinlocks used only on X86 */
28 KSPIN_LOCK KiFreezeExecutionLock;
29
30
31 KIPCR KiInitialPcr;
32
33 /* Boot and double-fault/NMI/DPC stack */
34 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
35 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
36 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
37 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
38
39 void KiInitializeSegments();
40 void KiSystemCallEntry64();
41 void KiSystemCallEntry32();
42
43 /* FUNCTIONS *****************************************************************/
44
45 VOID
46 NTAPI
47 KiInitMachineDependent(VOID)
48 {
49 /* Check for large page support */
50 if (KeFeatureBits & KF_LARGE_PAGE)
51 {
52 /* FIXME: Support this */
53 DPRINT("Large Page support detected but not yet taken advantage of!\n");
54 }
55
56 /* Check for global page support */
57 if (KeFeatureBits & KF_GLOBAL_PAGE)
58 {
59 /* FIXME: Support this */
60 DPRINT("Global Page support detected but not yet taken advantage of!\n");
61 }
62
63 /* Check if we have MTRR */
64 if (KeFeatureBits & KF_MTRR)
65 {
66 /* FIXME: Support this */
67 DPRINT("MTRR support detected but not yet taken advantage of!\n");
68 }
69
70 /* Check for PAT and/or MTRR support */
71 if (KeFeatureBits & KF_PAT)
72 {
73 /* FIXME: Support this */
74 DPRINT("PAT support detected but not yet taken advantage of!\n");
75 }
76
77 /* Allocate the IOPM save area. */
78 // Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
79 // PAGE_SIZE * 2,
80 // TAG('K', 'e', ' ', ' '));
81 // if (!Ki386IopmSaveArea)
82 // {
83 // /* Bugcheck. We need this for V86/VDM support. */
84 // KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
85 // }
86
87 /* Initialize 8/16 bit SList support */
88 RtlpUse16ByteSLists = (KeFeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
89 }
90
91 VOID
92 NTAPI
93 KiInitializePcr(IN PKIPCR Pcr,
94 IN ULONG ProcessorNumber,
95 IN PKTHREAD IdleThread,
96 IN PVOID DpcStack)
97 {
98 KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
99 PKGDTENTRY64 TssEntry;
100 USHORT Tr = 0;
101
102 /* Zero out the PCR */
103 RtlZeroMemory(Pcr, sizeof(KIPCR));
104
105 /* Set pointers to ourselves */
106 Pcr->Self = (PKPCR)Pcr;
107 Pcr->CurrentPrcb = &Pcr->Prcb;
108
109 /* Set the PCR Version */
110 Pcr->MajorVersion = PCR_MAJOR_VERSION;
111 Pcr->MinorVersion = PCR_MINOR_VERSION;
112
113 /* Set the PRCB Version */
114 Pcr->Prcb.MajorVersion = 1;
115 Pcr->Prcb.MinorVersion = 1;
116
117 /* Set the Build Type */
118 Pcr->Prcb.BuildType = 0;
119 #ifndef CONFIG_SMP
120 Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
121 #endif
122 #if DBG
123 Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
124 #endif
125
126 /* Set the Processor Number and current Processor Mask */
127 Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
128 Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
129
130 /* Get GDT and IDT descriptors */
131 __sgdt(&GdtDescriptor.Limit);
132 __sidt(&IdtDescriptor.Limit);
133 Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
134 Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
135
136 /* Get TSS Selector */
137 __str(&Tr);
138 ASSERT(Tr == KGDT64_SYS_TSS);
139
140 /* Get TSS Entry */
141 TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
142
143 /* Get the KTSS itself */
144 Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
145
146 Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
147
148 /* Set DPC Stack */
149 Pcr->Prcb.DpcStack = DpcStack;
150
151 /* Setup the processor set */
152 Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
153
154 /* Clear DR6/7 to cleanup bootloader debugging */
155 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
156 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
157
158 /* Set the Current Thread */
159 Pcr->Prcb.CurrentThread = IdleThread;
160
161 /* Start us out at PASSIVE_LEVEL */
162 Pcr->Irql = PASSIVE_LEVEL;
163 KeSetCurrentIrql(PASSIVE_LEVEL);
164 }
165
166 VOID
167 NTAPI
168 KiInitializeCpu(PKIPCR Pcr)
169 {
170 ULONG64 Pat;
171 ULONG FeatureBits;
172
173 /* Initialize gs */
174 KiInitializeSegments();
175
176 /* Set GS base */
177 __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
178 __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
179
180 /* Detect and set the CPU Type */
181 KiSetProcessorType();
182
183 /* Get the processor features for this CPU */
184 FeatureBits = KiGetFeatureBits();
185
186 /* Check if we support all needed features */
187 if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
188 {
189 /* If not, bugcheck system */
190 FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
191 FeatureBits, REQUIRED_FEATURE_BITS);
192 KeBugCheck(0);
193 }
194
195 /* Set DEP to always on */
196 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
197 FeatureBits |= KF_NX_ENABLED;
198
199 /* Save feature bits */
200 Pcr->Prcb.FeatureBits = FeatureBits;
201
202 /* Enable fx save restore support */
203 __writecr4(__readcr4() | CR4_FXSR);
204
205 /* Enable XMMI exceptions */
206 __writecr4(__readcr4() | CR4_XMMEXCPT);
207
208 /* Enable Write-Protection */
209 __writecr0(__readcr0() | CR0_WP);
210
211 /* Disable fpu monitoring */
212 __writecr0(__readcr0() & ~CR0_MP);
213
214 /* Disable x87 fpu exceptions */
215 __writecr0(__readcr0() & ~CR0_NE);
216
217 /* LDT is unused */
218 __lldt(0);
219
220 /* Set the systemcall entry points */
221 __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
222 __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
223
224 __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
225 ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
226
227 /* Set the flags to be cleared when doing a syscall */
228 __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
229
230 /* Enable syscall instruction and no-execute support */
231 __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
232
233 /* Initialize the PAT */
234 Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
235 (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
236 __writemsr(MSR_PAT, Pat);
237 }
238
239 VOID
240 FASTCALL
241 KiInitializeTss(IN PKTSS64 Tss,
242 IN UINT64 Stack)
243 {
244 PKGDTENTRY64 TssEntry;
245
246 /* Get pointer to the GDT entry */
247 TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
248
249 /* Initialize the GDT entry */
250 KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
251
252 /* Zero out the TSS */
253 RtlZeroMemory(Tss, sizeof(KTSS64));
254
255 /* FIXME: I/O Map? */
256 Tss->IoMapBase = 0x68;
257
258 /* Setup ring 0 stack pointer */
259 Tss->Rsp0 = Stack;
260
261 /* Setup a stack for Double Fault Traps */
262 Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
263
264 /* Setup a stack for CheckAbort Traps */
265 Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
266
267 /* Setup a stack for NMI Traps */
268 Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
269
270 /* Load the task register */
271 __ltr(KGDT64_SYS_TSS);
272 }
273
274 VOID
275 NTAPI
276 INIT_FUNCTION
277 KiInitializeKernelMachineDependent(
278 IN PKPRCB Prcb,
279 IN PLOADER_PARAMETER_BLOCK LoaderBlock)
280 {
281 /* Set boot-level flags */
282 KeI386CpuType = Prcb->CpuType;
283 KeI386CpuStep = Prcb->CpuStep;
284 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
285 KeProcessorLevel = (USHORT)Prcb->CpuType;
286 if (Prcb->CpuID)
287 KeProcessorRevision = Prcb->CpuStep;
288
289 /* Set basic CPU Features that user mode can read */
290 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
291 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
292 SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
293 SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
294 SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
295 SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
296 SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
297 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
298 (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
299 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
300 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
301 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
302 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
303 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
304 (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
305 SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
306 (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
307 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
308 (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
309
310 /* Set the default NX policy (opt-in) */
311 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
312
313 /* Check if NPX is always on */
314 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
315 {
316 /* Set it always on */
317 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
318 Prcb->FeatureBits |= KF_NX_ENABLED;
319 }
320 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
321 {
322 /* Set it in opt-out mode */
323 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
324 Prcb->FeatureBits |= KF_NX_ENABLED;
325 }
326 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
327 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
328 {
329 /* Set the feature bits */
330 Prcb->FeatureBits |= KF_NX_ENABLED;
331 }
332 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
333 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
334 {
335 /* Set disabled mode */
336 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
337 Prcb->FeatureBits |= KF_NX_DISABLED;
338 }
339 }
340
341 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
342
343 void
344 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
345 {
346 PLDR_DATA_TABLE_ENTRY LdrEntry;
347 PLIST_ENTRY Entry;
348 ULONG i;
349
350 /* Initialize the list head */
351 InitializeListHead(&PsLoadedModuleList);
352
353 /* Loop the first 3 entries */
354 for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
355 Entry != &LoaderBlock->LoadOrderListHead && i < 3;
356 Entry = Entry->Flink, i++)
357 {
358 /* Get the data table entry */
359 LdrEntry = CONTAINING_RECORD(Entry,
360 LDR_DATA_TABLE_ENTRY,
361 InLoadOrderLinks);
362
363 /* Copy the entry */
364 LdrCoreEntries[i] = *LdrEntry;
365
366 /* Insert the copy into the list */
367 InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
368 }
369 }
370
371 VOID
372 NTAPI
373 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
374 {
375 CCHAR Cpu;
376 PKTHREAD InitialThread;
377 ULONG64 InitialStack;
378 PKIPCR Pcr;
379
380 /* HACK */
381 FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
382 //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
383
384 /* Save the loader block */
385 KeLoaderBlock = LoaderBlock;
386
387 /* Get the current CPU number */
388 Cpu = KeNumberProcessors++; // FIXME
389
390 /* LoaderBlock initialization for Cpu 0 */
391 if (Cpu == 0)
392 {
393 /* Set the initial stack, idle thread and process */
394 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
395 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
396 LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
397 LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
398 }
399
400 /* Get Pcr from loader block */
401 Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
402
403 /* Set the PRCB for this Processor */
404 KiProcessorBlock[Cpu] = &Pcr->Prcb;
405
406 /* Align stack to 16 bytes */
407 LoaderBlock->KernelStack &= ~(16 - 1);
408
409 /* Save the initial thread and stack */
410 InitialStack = LoaderBlock->KernelStack; // Checkme
411 InitialThread = (PKTHREAD)LoaderBlock->Thread;
412
413 /* Set us as the current process */
414 InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
415
416 /* Initialize the PCR */
417 KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
418
419 /* Initialize the CPU features */
420 KiInitializeCpu(Pcr);
421
422 /* Initial setup for the boot CPU */
423 if (Cpu == 0)
424 {
425 /* Initialize the module list (ntos, hal, kdcom) */
426 KiInitModuleList(LoaderBlock);
427
428 /* Setup the TSS descriptors and entries */
429 KiInitializeTss(Pcr->TssBase, InitialStack);
430
431 /* Setup the IDT */
432 KeInitExceptions();
433
434 /* Initialize debugging system */
435 KdInitSystem(0, KeLoaderBlock);
436
437 /* Check for break-in */
438 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
439 }
440
441 DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
442 Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
443
444 /* Acquire lock */
445 while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
446 {
447 /* Loop until lock is free */
448 while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
449 }
450
451 /* Initialize the Processor with HAL */
452 HalInitializeProcessor(Cpu, KeLoaderBlock);
453
454 /* Set processor as active */
455 KeActiveProcessors |= 1ULL << Cpu;
456
457 /* Release lock */
458 InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
459
460 /* Raise to HIGH_LEVEL */
461 KfRaiseIrql(HIGH_LEVEL);
462
463 /* Machine specific kernel initialization */
464 if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
465
466 /* Switch to new kernel stack and start kernel bootstrapping */
467 KiSwitchToBootStack(InitialStack & ~3);
468 }
469