ntoskrnl/ke/amd64/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
  17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
  18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT)
  19
  20 /* GLOBALS *******************************************************************/
  21
  22 /* Function pointer for early debug prints */
  23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
  24
  25 /* Spinlocks used only on X86 */
  26 KSPIN_LOCK KiFreezeExecutionLock;
  27
  28
  29 KIPCR KiInitialPcr;
  30
  31 /* Boot and double-fault/NMI/DPC stack */
  32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  36
  37 void KiInitializeSegments();
  38 void KiSystemCallEntry64();
  39 void KiSystemCallEntry32();
  40
  41 /* FUNCTIONS *****************************************************************/
  42
  43 VOID
  44 NTAPI
  45 KiInitMachineDependent(VOID)
  46 {
  47     /* Check for large page support */
  48     if (KeFeatureBits & KF_LARGE_PAGE)
  49     {
  50         /* FIXME: Support this */
  51         DPRINT("Large Page support detected but not yet taken advantage of!\n");
  52     }
  53
  54     /* Check for global page support */
  55     if (KeFeatureBits & KF_GLOBAL_PAGE)
  56     {
  57         /* FIXME: Support this */
  58         DPRINT("Global Page support detected but not yet taken advantage of!\n");
  59     }
  60
  61     /* Check if we have MTRR */
  62     if (KeFeatureBits & KF_MTRR)
  63     {
  64         /* FIXME: Support this */
  65         DPRINT("MTRR support detected but not yet taken advantage of!\n");
  66     }
  67
  68     /* Check for PAT and/or MTRR support */
  69     if (KeFeatureBits & KF_PAT)
  70     {
  71         /* FIXME: Support this */
  72         DPRINT("PAT support detected but not yet taken advantage of!\n");
  73     }
  74
  75         /* Allocate the IOPM save area. */
  76 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
  77 //                                                  PAGE_SIZE * 2,
  78 //                                                  TAG('K', 'e', ' ', ' '));
  79 //        if (!Ki386IopmSaveArea)
  80 //        {
  81 //            /* Bugcheck. We need this for V86/VDM support. */
  82 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
  83 //        }
  84
  85 }
  86
  87 VOID
  88 NTAPI
  89 KiInitializePcr(IN PKIPCR Pcr,
  90                 IN ULONG ProcessorNumber,
  91                 IN PKTHREAD IdleThread,
  92                 IN PVOID DpcStack)
  93 {
  94     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
  95     PKGDTENTRY64 TssEntry;
  96     USHORT Tr = 0;
  97
  98     /* Zero out the PCR */
  99     RtlZeroMemory(Pcr, sizeof(KIPCR));
 100
 101     /* Set pointers to ourselves */
 102     Pcr->Self = (PKPCR)Pcr;
 103     Pcr->CurrentPrcb = &Pcr->Prcb;
 104
 105     /* Set the PCR Version */
 106     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 107     Pcr->MinorVersion = PCR_MINOR_VERSION;
 108
 109     /* Set the PRCB Version */
 110     Pcr->Prcb.MajorVersion = 1;
 111     Pcr->Prcb.MinorVersion = 1;
 112
 113     /* Set the Build Type */
 114     Pcr->Prcb.BuildType = 0;
 115 #ifndef CONFIG_SMP
 116     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 117 #endif
 118 #if DBG
 119     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
 120 #endif
 121
 122     /* Set the Processor Number and current Processor Mask */
 123     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
 124     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
 125
 126     /* Get GDT and IDT descriptors */
 127     __sgdt(&GdtDescriptor.Limit);
 128     __sidt(&IdtDescriptor.Limit);
 129     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
 130     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
 131
 132     /* Get TSS Selector */
 133     __str(&Tr);
 134     ASSERT(Tr == KGDT64_SYS_TSS);
 135
 136     /* Get TSS Entry */
 137     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
 138
 139     /* Get the KTSS itself */
 140     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
 141
 142     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
 143
 144     /* Set DPC Stack */
 145     Pcr->Prcb.DpcStack = DpcStack;
 146
 147     /* Setup the processor set */
 148     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
 149
 150     /* Clear DR6/7 to cleanup bootloader debugging */
 151     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
 152     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
 153
 154     /* Set the Current Thread */
 155     Pcr->Prcb.CurrentThread = IdleThread;
 156
 157     /* Start us out at PASSIVE_LEVEL */
 158     Pcr->Irql = PASSIVE_LEVEL;
 159     KeSetCurrentIrql(PASSIVE_LEVEL);
 160 }
 161
 162 VOID
 163 NTAPI
 164 KiInitializeCpu(PKIPCR Pcr)
 165 {
 166     ULONG64 Pat;
 167     ULONG FeatureBits;
 168
 169     /* Initialize gs */
 170     KiInitializeSegments();
 171
 172     /* Set GS base */
 173     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
 174     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
 175
 176     /* Detect and set the CPU Type */
 177     KiSetProcessorType();
 178
 179     /* Get the processor features for this CPU */
 180     FeatureBits = KiGetFeatureBits();
 181
 182     /* Check if we support all needed features */
 183     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
 184     {
 185         /* If not, bugcheck system */
 186         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
 187                 FeatureBits, REQUIRED_FEATURE_BITS);
 188         KeBugCheck(0);
 189     }
 190
 191     /* Set DEP to always on */
 192     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 193     FeatureBits |= KF_NX_ENABLED;
 194
 195     /* Save feature bits */
 196     Pcr->Prcb.FeatureBits = FeatureBits;
 197
 198     /* Enable fx save restore support */
 199     __writecr4(__readcr4() | CR4_FXSR);
 200
 201     /* Enable XMMI exceptions */
 202     __writecr4(__readcr4() | CR4_XMMEXCPT);
 203
 204     /* Enable Write-Protection */
 205     __writecr0(__readcr0() | CR0_WP);
 206
 207     /* Disable fpu monitoring */
 208     __writecr0(__readcr0() & ~CR0_MP);
 209
 210     /* Disable x87 fpu exceptions */
 211     __writecr0(__readcr0() & ~CR0_NE);
 212
 213     /* LDT is unused */
 214     __lldt(0);
 215
 216     /* Set the systemcall entry points */
 217     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
 218     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
 219
 220     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
 221                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
 222
 223     /* Set the flags to be cleared when doing a syscall */
 224     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
 225
 226     /* Enable syscall instruction and no-execute support */
 227     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
 228
 229     /* Initialize the PAT */
 230     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
 231           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
 232     __writemsr(MSR_PAT, Pat);
 233 }
 234
 235 VOID
 236 FASTCALL
 237 KiInitializeTss(IN PKTSS64 Tss,
 238                 IN UINT64 Stack)
 239 {
 240     PKGDTENTRY64 TssEntry;
 241
 242     /* Get pointer to the GDT entry */
 243     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
 244
 245     /* Initialize the GDT entry */
 246     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
 247
 248     /* Zero out the TSS */
 249     RtlZeroMemory(Tss, sizeof(KTSS64));
 250
 251     /* FIXME: I/O Map? */
 252     Tss->IoMapBase = 0x68;
 253
 254     /* Setup ring 0 stack pointer */
 255     Tss->Rsp0 = Stack;
 256
 257     /* Setup a stack for Double Fault Traps */
 258     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
 259
 260     /* Setup a stack for CheckAbort Traps */
 261     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
 262
 263     /* Setup a stack for NMI Traps */
 264     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
 265
 266     /* Load the task register */
 267     __ltr(KGDT64_SYS_TSS);
 268 }
 269
 270 VOID
 271 NTAPI
 272 INIT_FUNCTION
 273 KiInitializeKernelMachineDependent(
 274     IN PKPRCB Prcb,
 275     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 276 {
 277     /* Set boot-level flags */
 278     KeI386CpuType = Prcb->CpuType;
 279     KeI386CpuStep = Prcb->CpuStep;
 280     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
 281     KeProcessorLevel = (USHORT)Prcb->CpuType;
 282     if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
 283
 284     /* Set basic CPU Features that user mode can read */
 285     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 286         (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
 287     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
 288     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 289         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
 290     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 291         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 292     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 293         (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
 294     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
 295
 296     /* Set the default NX policy (opt-in) */
 297     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 298
 299     /* Check if NPX is always on */
 300     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 301     {
 302         /* Set it always on */
 303         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 304         Prcb->FeatureBits |= KF_NX_ENABLED;
 305     }
 306     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 307     {
 308         /* Set it in opt-out mode */
 309         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 310         Prcb->FeatureBits |= KF_NX_ENABLED;
 311     }
 312     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 313              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 314     {
 315         /* Set the feature bits */
 316         Prcb->FeatureBits |= KF_NX_ENABLED;
 317     }
 318     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 319              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 320     {
 321         /* Set disabled mode */
 322         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 323         Prcb->FeatureBits |= KF_NX_DISABLED;
 324     }
 325
 326 }
 327
 328 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
 329
 330 void
 331 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 332 {
 333     PLDR_DATA_TABLE_ENTRY LdrEntry;
 334     PLIST_ENTRY Entry;
 335     ULONG i;
 336
 337     /* Initialize the list head */
 338     InitializeListHead(&PsLoadedModuleList);
 339
 340     /* Loop the first 3 entries */
 341     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
 342          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
 343          Entry = Entry->Flink, i++)
 344     {
 345         /* Get the data table entry */
 346         LdrEntry = CONTAINING_RECORD(Entry,
 347                                      LDR_DATA_TABLE_ENTRY,
 348                                      InLoadOrderLinks);
 349
 350         /* Copy the entry */
 351         LdrCoreEntries[i] = *LdrEntry;
 352
 353         /* Insert the copy into the list */
 354         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
 355     }
 356 }
 357
 358 VOID
 359 NTAPI
 360 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 361 {
 362     CCHAR Cpu;
 363     PKTHREAD InitialThread;
 364     ULONG64 InitialStack;
 365     PKIPCR Pcr;
 366
 367     /* HACK */
 368     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
 369     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
 370
 371     /* Save the loader block */
 372     KeLoaderBlock = LoaderBlock;
 373
 374     /* Get the current CPU number */
 375     Cpu = (CCHAR)KeNumberProcessors++; // FIXME
 376
 377     /* LoaderBlock initialization for Cpu 0 */
 378     if (Cpu == 0)
 379     {
 380         /* Set the initial stack, idle thread and process */
 381         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 382         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 383         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
 384         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
 385     }
 386
 387     /* Get Pcr from loader block */
 388     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
 389
 390     /* Set the PRCB for this Processor */
 391     KiProcessorBlock[Cpu] = &Pcr->Prcb;
 392
 393     /* Align stack to 16 bytes */
 394     LoaderBlock->KernelStack &= ~(16 - 1);
 395
 396     /* Save the initial thread and stack */
 397     InitialStack = LoaderBlock->KernelStack; // Checkme
 398     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 399
 400     /* Set us as the current process */
 401     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
 402
 403     /* Initialize the PCR */
 404     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
 405
 406     /* Initialize the CPU features */
 407     KiInitializeCpu(Pcr);
 408
 409     /* Initial setup for the boot CPU */
 410     if (Cpu == 0)
 411     {
 412         /* Initialize the module list (ntos, hal, kdcom) */
 413         KiInitModuleList(LoaderBlock);
 414
 415         /* Setup the TSS descriptors and entries */
 416         KiInitializeTss(Pcr->TssBase, InitialStack);
 417
 418         /* Setup the IDT */
 419         KeInitExceptions();
 420
 421          /* Initialize debugging system */
 422         KdInitSystem(0, KeLoaderBlock);
 423
 424         /* Check for break-in */
 425         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 426     }
 427
 428     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
 429            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
 430
 431     /* Acquire lock */
 432     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
 433     {
 434         /* Loop until lock is free */
 435         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
 436     }
 437
 438     /* Initialize the Processor with HAL */
 439     HalInitializeProcessor(Cpu, KeLoaderBlock);
 440
 441     /* Set processor as active */
 442     KeActiveProcessors |= 1ULL << Cpu;
 443
 444     /* Release lock */
 445     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
 446
 447     /* Raise to HIGH_LEVEL */
 448     KfRaiseIrql(HIGH_LEVEL);
 449
 450     /* Machine specific kernel initialization */
 451     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
 452
 453     /* Switch to new kernel stack and start kernel bootstrapping */
 454     KiSwitchToBootStack(InitialStack & ~3);
 455 }
 456