[TRANSLATION] Chinese Traditional translation improvements (#3085)
[reactos.git] / ntoskrnl / ke / amd64 / kiinit.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/amd64/kiinit.c
5 * PURPOSE: Kernel Initialization for x86 CPUs
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Timo Kreuzer (timo.kreuzer@reactos.org)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
17 KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
18 KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
19
20 /* GLOBALS *******************************************************************/
21
22 /* Function pointer for early debug prints */
23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
24
25 /* Spinlocks used only on X86 */
26 KSPIN_LOCK KiFreezeExecutionLock;
27
28
29 KIPCR KiInitialPcr;
30
31 /* Boot and double-fault/NMI/DPC stack */
32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
36
37 void KiInitializeSegments();
38 void KiSystemCallEntry64();
39 void KiSystemCallEntry32();
40
41 /* FUNCTIONS *****************************************************************/
42
43 INIT_FUNCTION
44 VOID
45 NTAPI
46 KiInitMachineDependent(VOID)
47 {
48 /* Check for large page support */
49 if (KeFeatureBits & KF_LARGE_PAGE)
50 {
51 /* FIXME: Support this */
52 DPRINT("Large Page support detected but not yet taken advantage of!\n");
53 }
54
55 /* Check for global page support */
56 if (KeFeatureBits & KF_GLOBAL_PAGE)
57 {
58 /* FIXME: Support this */
59 DPRINT("Global Page support detected but not yet taken advantage of!\n");
60 }
61
62 /* Check if we have MTRR */
63 if (KeFeatureBits & KF_MTRR)
64 {
65 /* FIXME: Support this */
66 DPRINT("MTRR support detected but not yet taken advantage of!\n");
67 }
68
69 /* Check for PAT and/or MTRR support */
70 if (KeFeatureBits & KF_PAT)
71 {
72 /* FIXME: Support this */
73 DPRINT("PAT support detected but not yet taken advantage of!\n");
74 }
75
76 // /* Allocate the IOPM save area */
77 // Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
78 // IOPM_SIZE,
79 // ' eK');
80 // if (!Ki386IopmSaveArea)
81 // {
82 // /* Bugcheck. We need this for V86/VDM support. */
83 // KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0);
84 // }
85
86 }
87
88 VOID
89 NTAPI
90 KiInitializePcr(IN PKIPCR Pcr,
91 IN ULONG ProcessorNumber,
92 IN PKTHREAD IdleThread,
93 IN PVOID DpcStack)
94 {
95 KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
96 PKGDTENTRY64 TssEntry;
97 USHORT Tr = 0;
98
99 /* Zero out the PCR */
100 RtlZeroMemory(Pcr, sizeof(KIPCR));
101
102 /* Set pointers to ourselves */
103 Pcr->Self = (PKPCR)Pcr;
104 Pcr->CurrentPrcb = &Pcr->Prcb;
105
106 /* Set the PCR Version */
107 Pcr->MajorVersion = PCR_MAJOR_VERSION;
108 Pcr->MinorVersion = PCR_MINOR_VERSION;
109
110 /* Set the PRCB Version */
111 Pcr->Prcb.MajorVersion = 1;
112 Pcr->Prcb.MinorVersion = 1;
113
114 /* Set the Build Type */
115 Pcr->Prcb.BuildType = 0;
116 #ifndef CONFIG_SMP
117 Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
118 #endif
119 #if DBG
120 Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
121 #endif
122
123 /* Set the Processor Number and current Processor Mask */
124 Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
125 Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
126
127 /* Get GDT and IDT descriptors */
128 __sgdt(&GdtDescriptor.Limit);
129 __sidt(&IdtDescriptor.Limit);
130 Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
131 Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
132
133 /* Get TSS Selector */
134 __str(&Tr);
135 ASSERT(Tr == KGDT64_SYS_TSS);
136
137 /* Get TSS Entry */
138 TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
139
140 /* Get the KTSS itself */
141 Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
142
143 Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
144
145 /* Set DPC Stack */
146 Pcr->Prcb.DpcStack = DpcStack;
147
148 /* Setup the processor set */
149 Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
150
151 /* Clear DR6/7 to cleanup bootloader debugging */
152 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
153 Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
154
155 /* Set the Current Thread */
156 Pcr->Prcb.CurrentThread = IdleThread;
157
158 /* Start us out at PASSIVE_LEVEL */
159 Pcr->Irql = PASSIVE_LEVEL;
160 KeSetCurrentIrql(PASSIVE_LEVEL);
161 }
162
163 VOID
164 NTAPI
165 KiInitializeCpu(PKIPCR Pcr)
166 {
167 ULONG64 Pat;
168 ULONG FeatureBits;
169
170 /* Initialize gs */
171 KiInitializeSegments();
172
173 /* Set GS base */
174 __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
175 __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
176
177 /* Detect and set the CPU Type */
178 KiSetProcessorType();
179
180 /* Get the processor features for this CPU */
181 FeatureBits = KiGetFeatureBits();
182
183 /* Check if we support all needed features */
184 if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
185 {
186 /* If not, bugcheck system */
187 FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
188 FeatureBits, REQUIRED_FEATURE_BITS);
189 KeBugCheck(0);
190 }
191
192 /* Set DEP to always on */
193 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
194 FeatureBits |= KF_NX_ENABLED;
195
196 /* Save feature bits */
197 Pcr->Prcb.FeatureBits = FeatureBits;
198
199 /* Enable fx save restore support */
200 __writecr4(__readcr4() | CR4_FXSR);
201
202 /* Enable XMMI exceptions */
203 __writecr4(__readcr4() | CR4_XMMEXCPT);
204
205 /* Enable Write-Protection */
206 __writecr0(__readcr0() | CR0_WP);
207
208 /* Disable fpu monitoring */
209 __writecr0(__readcr0() & ~CR0_MP);
210
211 /* Disable x87 fpu exceptions */
212 __writecr0(__readcr0() & ~CR0_NE);
213
214 /* LDT is unused */
215 __lldt(0);
216
217 /* Set the systemcall entry points */
218 __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
219 __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
220
221 __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
222 ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
223
224 /* Set the flags to be cleared when doing a syscall */
225 __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
226
227 /* Enable syscall instruction and no-execute support */
228 __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
229
230 /* Initialize the PAT */
231 Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
232 (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
233 __writemsr(MSR_PAT, Pat);
234 }
235
236 VOID
237 FASTCALL
238 KiInitializeTss(IN PKTSS64 Tss,
239 IN UINT64 Stack)
240 {
241 PKGDTENTRY64 TssEntry;
242
243 /* Get pointer to the GDT entry */
244 TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
245
246 /* Initialize the GDT entry */
247 KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
248
249 /* Zero out the TSS */
250 RtlZeroMemory(Tss, sizeof(KTSS64));
251
252 /* FIXME: I/O Map? */
253 Tss->IoMapBase = 0x68;
254
255 /* Setup ring 0 stack pointer */
256 Tss->Rsp0 = Stack;
257
258 /* Setup a stack for Double Fault Traps */
259 Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
260
261 /* Setup a stack for CheckAbort Traps */
262 Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
263
264 /* Setup a stack for NMI Traps */
265 Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
266
267 /* Load the task register */
268 __ltr(KGDT64_SYS_TSS);
269 }
270
271 INIT_FUNCTION
272 VOID
273 NTAPI
274 KiInitializeKernelMachineDependent(
275 IN PKPRCB Prcb,
276 IN PLOADER_PARAMETER_BLOCK LoaderBlock)
277 {
278 /* Set boot-level flags */
279 KeI386CpuType = Prcb->CpuType;
280 KeI386CpuStep = Prcb->CpuStep;
281 KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
282 KeProcessorLevel = (USHORT)Prcb->CpuType;
283 if (Prcb->CpuID)
284 KeProcessorRevision = Prcb->CpuStep;
285
286 /* Set basic CPU Features that user mode can read */
287 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
288 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
289 SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
290 SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
291 SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
292 SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
293 SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
294 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
295 (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
296 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
297 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
298 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
299 ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
300 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
301 (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
302 SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
303 (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
304 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
305 (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
306
307 /* Set the default NX policy (opt-in) */
308 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
309
310 /* Check if NPX is always on */
311 if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
312 {
313 /* Set it always on */
314 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
315 Prcb->FeatureBits |= KF_NX_ENABLED;
316 }
317 else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
318 {
319 /* Set it in opt-out mode */
320 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
321 Prcb->FeatureBits |= KF_NX_ENABLED;
322 }
323 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
324 (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
325 {
326 /* Set the feature bits */
327 Prcb->FeatureBits |= KF_NX_ENABLED;
328 }
329 else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
330 (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
331 {
332 /* Set disabled mode */
333 SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
334 Prcb->FeatureBits |= KF_NX_DISABLED;
335 }
336 }
337
338 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
339
340 void
341 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
342 {
343 PLDR_DATA_TABLE_ENTRY LdrEntry;
344 PLIST_ENTRY Entry;
345 ULONG i;
346
347 /* Initialize the list head */
348 InitializeListHead(&PsLoadedModuleList);
349
350 /* Loop the first 3 entries */
351 for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
352 Entry != &LoaderBlock->LoadOrderListHead && i < 3;
353 Entry = Entry->Flink, i++)
354 {
355 /* Get the data table entry */
356 LdrEntry = CONTAINING_RECORD(Entry,
357 LDR_DATA_TABLE_ENTRY,
358 InLoadOrderLinks);
359
360 /* Copy the entry */
361 LdrCoreEntries[i] = *LdrEntry;
362
363 /* Insert the copy into the list */
364 InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
365 }
366 }
367
368 INIT_FUNCTION
369 VOID
370 NTAPI
371 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
372 {
373 CCHAR Cpu;
374 PKTHREAD InitialThread;
375 ULONG64 InitialStack;
376 PKIPCR Pcr;
377
378 /* HACK */
379 FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
380 //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
381
382 /* Save the loader block */
383 KeLoaderBlock = LoaderBlock;
384
385 /* Get the current CPU number */
386 Cpu = KeNumberProcessors++; // FIXME
387
388 /* LoaderBlock initialization for Cpu 0 */
389 if (Cpu == 0)
390 {
391 /* Set the initial stack, idle thread and process */
392 LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
393 LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
394 LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
395 LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
396 }
397
398 /* Get Pcr from loader block */
399 Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
400
401 /* Set the PRCB for this Processor */
402 KiProcessorBlock[Cpu] = &Pcr->Prcb;
403
404 /* Align stack to 16 bytes */
405 LoaderBlock->KernelStack &= ~(16 - 1);
406
407 /* Save the initial thread and stack */
408 InitialStack = LoaderBlock->KernelStack; // Checkme
409 InitialThread = (PKTHREAD)LoaderBlock->Thread;
410
411 /* Set us as the current process */
412 InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
413
414 /* Initialize the PCR */
415 KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
416
417 /* Initialize the CPU features */
418 KiInitializeCpu(Pcr);
419
420 /* Initial setup for the boot CPU */
421 if (Cpu == 0)
422 {
423 /* Initialize the module list (ntos, hal, kdcom) */
424 KiInitModuleList(LoaderBlock);
425
426 /* Setup the TSS descriptors and entries */
427 KiInitializeTss(Pcr->TssBase, InitialStack);
428
429 /* Setup the IDT */
430 KeInitExceptions();
431
432 /* Initialize debugging system */
433 KdInitSystem(0, KeLoaderBlock);
434
435 /* Check for break-in */
436 if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
437 }
438
439 DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
440 Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
441
442 /* Acquire lock */
443 while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
444 {
445 /* Loop until lock is free */
446 while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
447 }
448
449 /* Initialize the Processor with HAL */
450 HalInitializeProcessor(Cpu, KeLoaderBlock);
451
452 /* Set processor as active */
453 KeActiveProcessors |= 1ULL << Cpu;
454
455 /* Release lock */
456 InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
457
458 /* Raise to HIGH_LEVEL */
459 KfRaiseIrql(HIGH_LEVEL);
460
461 /* Machine specific kernel initialization */
462 if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
463
464 /* Switch to new kernel stack and start kernel bootstrapping */
465 KiSwitchToBootStack(InitialStack & ~3);
466 }
467