reactos/ntoskrnl/ke/amd64/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/i386/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
  17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
  18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
  19
  20 /* GLOBALS *******************************************************************/
  21
  22 extern BOOLEAN RtlpUse16ByteSLists;
  23
  24 /* Function pointer for early debug prints */
  25 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
  26
  27 /* Spinlocks used only on X86 */
  28 KSPIN_LOCK KiFreezeExecutionLock;
  29
  30
  31 KIPCR KiInitialPcr;
  32
  33 /* Boot and double-fault/NMI/DPC stack */
  34 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  35 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  36 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  37 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  38
  39 void KiInitializeSegments();
  40 void KiSystemCallEntry64();
  41 void KiSystemCallEntry32();
  42
  43 /* FUNCTIONS *****************************************************************/
  44
  45 VOID
  46 NTAPI
  47 KiInitMachineDependent(VOID)
  48 {
  49     /* Check for large page support */
  50     if (KeFeatureBits & KF_LARGE_PAGE)
  51     {
  52         /* FIXME: Support this */
  53         DPRINT("Large Page support detected but not yet taken advantage of!\n");
  54     }
  55
  56     /* Check for global page support */
  57     if (KeFeatureBits & KF_GLOBAL_PAGE)
  58     {
  59         /* FIXME: Support this */
  60         DPRINT("Global Page support detected but not yet taken advantage of!\n");
  61     }
  62
  63     /* Check if we have MTRR */
  64     if (KeFeatureBits & KF_MTRR)
  65     {
  66         /* FIXME: Support this */
  67         DPRINT("MTRR support detected but not yet taken advantage of!\n");
  68     }
  69
  70     /* Check for PAT and/or MTRR support */
  71     if (KeFeatureBits & KF_PAT)
  72     {
  73         /* FIXME: Support this */
  74         DPRINT("PAT support detected but not yet taken advantage of!\n");
  75     }
  76
  77         /* Allocate the IOPM save area. */
  78 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
  79 //                                                  PAGE_SIZE * 2,
  80 //                                                  TAG('K', 'e', ' ', ' '));
  81 //        if (!Ki386IopmSaveArea)
  82 //        {
  83 //            /* Bugcheck. We need this for V86/VDM support. */
  84 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
  85 //        }
  86
  87     /* Initialize 8/16 bit SList support */
  88     RtlpUse16ByteSLists = (KeFeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
  89 }
  90
  91 VOID
  92 NTAPI
  93 KiInitializePcr(IN PKIPCR Pcr,
  94                 IN ULONG ProcessorNumber,
  95                 IN PKTHREAD IdleThread,
  96                 IN PVOID DpcStack)
  97 {
  98     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
  99     PKGDTENTRY64 TssEntry;
 100     USHORT Tr = 0;
 101
 102     /* Zero out the PCR */
 103     RtlZeroMemory(Pcr, sizeof(KIPCR));
 104
 105     /* Set pointers to ourselves */
 106     Pcr->Self = (PKPCR)Pcr;
 107     Pcr->CurrentPrcb = &Pcr->Prcb;
 108
 109     /* Set the PCR Version */
 110     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 111     Pcr->MinorVersion = PCR_MINOR_VERSION;
 112
 113     /* Set the PRCB Version */
 114     Pcr->Prcb.MajorVersion = 1;
 115     Pcr->Prcb.MinorVersion = 1;
 116
 117     /* Set the Build Type */
 118     Pcr->Prcb.BuildType = 0;
 119 #ifndef CONFIG_SMP
 120     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 121 #endif
 122 #if DBG
 123     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
 124 #endif
 125
 126     /* Set the Processor Number and current Processor Mask */
 127     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
 128     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
 129
 130     /* Get GDT and IDT descriptors */
 131     __sgdt(&GdtDescriptor.Limit);
 132     __sidt(&IdtDescriptor.Limit);
 133     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
 134     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
 135
 136     /* Get TSS Selector */
 137     __str(&Tr);
 138     ASSERT(Tr == KGDT64_SYS_TSS);
 139
 140     /* Get TSS Entry */
 141     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
 142
 143     /* Get the KTSS itself */
 144     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
 145
 146     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
 147
 148     /* Set DPC Stack */
 149     Pcr->Prcb.DpcStack = DpcStack;
 150
 151     /* Setup the processor set */
 152     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
 153
 154     /* Clear DR6/7 to cleanup bootloader debugging */
 155     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
 156     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
 157
 158     /* Set the Current Thread */
 159     Pcr->Prcb.CurrentThread = IdleThread;
 160
 161     /* Start us out at PASSIVE_LEVEL */
 162     Pcr->Irql = PASSIVE_LEVEL;
 163     KeSetCurrentIrql(PASSIVE_LEVEL);
 164 }
 165
 166 VOID
 167 NTAPI
 168 KiInitializeCpu(PKIPCR Pcr)
 169 {
 170     ULONG64 Pat;
 171     ULONG FeatureBits;
 172
 173     /* Initialize gs */
 174     KiInitializeSegments();
 175
 176     /* Set GS base */
 177     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
 178     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
 179
 180     /* Detect and set the CPU Type */
 181     KiSetProcessorType();
 182
 183     /* Get the processor features for this CPU */
 184     FeatureBits = KiGetFeatureBits();
 185
 186     /* Check if we support all needed features */
 187     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
 188     {
 189         /* If not, bugcheck system */
 190         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
 191                 FeatureBits, REQUIRED_FEATURE_BITS);
 192         KeBugCheck(0);
 193     }
 194
 195     /* Set DEP to always on */
 196     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 197     FeatureBits |= KF_NX_ENABLED;
 198
 199     /* Save feature bits */
 200     Pcr->Prcb.FeatureBits = FeatureBits;
 201
 202     /* Enable fx save restore support */
 203     __writecr4(__readcr4() | CR4_FXSR);
 204
 205     /* Enable XMMI exceptions */
 206     __writecr4(__readcr4() | CR4_XMMEXCPT);
 207
 208     /* Enable Write-Protection */
 209     __writecr0(__readcr0() | CR0_WP);
 210
 211     /* Disable fpu monitoring */
 212     __writecr0(__readcr0() & ~CR0_MP);
 213
 214     /* Disable x87 fpu exceptions */
 215     __writecr0(__readcr0() & ~CR0_NE);
 216
 217     /* LDT is unused */
 218     __lldt(0);
 219
 220     /* Set the systemcall entry points */
 221     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
 222     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
 223
 224     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
 225                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
 226
 227     /* Set the flags to be cleared when doing a syscall */
 228     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
 229
 230     /* Enable syscall instruction and no-execute support */
 231     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
 232
 233     /* Initialize the PAT */
 234     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
 235           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
 236     __writemsr(MSR_PAT, Pat);
 237 }
 238
 239 VOID
 240 FASTCALL
 241 KiInitializeTss(IN PKTSS64 Tss,
 242                 IN UINT64 Stack)
 243 {
 244     PKGDTENTRY64 TssEntry;
 245
 246     /* Get pointer to the GDT entry */
 247     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
 248
 249     /* Initialize the GDT entry */
 250     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
 251
 252     /* Zero out the TSS */
 253     RtlZeroMemory(Tss, sizeof(KTSS64));
 254
 255     /* FIXME: I/O Map? */
 256     Tss->IoMapBase = 0x68;
 257
 258     /* Setup ring 0 stack pointer */
 259     Tss->Rsp0 = Stack;
 260
 261     /* Setup a stack for Double Fault Traps */
 262     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
 263
 264     /* Setup a stack for CheckAbort Traps */
 265     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
 266
 267     /* Setup a stack for NMI Traps */
 268     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
 269
 270     /* Load the task register */
 271     __ltr(KGDT64_SYS_TSS);
 272 }
 273
 274 VOID
 275 NTAPI
 276 INIT_FUNCTION
 277 KiInitializeKernelMachineDependent(
 278     IN PKPRCB Prcb,
 279     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 280 {
 281     /* Set boot-level flags */
 282     KeI386CpuType = Prcb->CpuType;
 283     KeI386CpuStep = Prcb->CpuStep;
 284     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
 285     KeProcessorLevel = (USHORT)Prcb->CpuType;
 286     if (Prcb->CpuID)
 287         KeProcessorRevision = Prcb->CpuStep;
 288
 289     /* Set basic CPU Features that user mode can read */
 290     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
 291     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
 292     SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
 293     SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
 294     SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
 295     SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
 296     SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
 297     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 298         (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
 299     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 300         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
 301     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 302         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 303     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 304         (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
 305     SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
 306         (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
 307     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
 308         (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
 309
 310     /* Set the default NX policy (opt-in) */
 311     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 312
 313     /* Check if NPX is always on */
 314     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 315     {
 316         /* Set it always on */
 317         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 318         Prcb->FeatureBits |= KF_NX_ENABLED;
 319     }
 320     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 321     {
 322         /* Set it in opt-out mode */
 323         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 324         Prcb->FeatureBits |= KF_NX_ENABLED;
 325     }
 326     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 327              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 328     {
 329         /* Set the feature bits */
 330         Prcb->FeatureBits |= KF_NX_ENABLED;
 331     }
 332     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 333              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 334     {
 335         /* Set disabled mode */
 336         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 337         Prcb->FeatureBits |= KF_NX_DISABLED;
 338     }
 339 }
 340
 341 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
 342
 343 void
 344 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 345 {
 346     PLDR_DATA_TABLE_ENTRY LdrEntry;
 347     PLIST_ENTRY Entry;
 348     ULONG i;
 349
 350     /* Initialize the list head */
 351     InitializeListHead(&PsLoadedModuleList);
 352
 353     /* Loop the first 3 entries */
 354     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
 355          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
 356          Entry = Entry->Flink, i++)
 357     {
 358         /* Get the data table entry */
 359         LdrEntry = CONTAINING_RECORD(Entry,
 360                                      LDR_DATA_TABLE_ENTRY,
 361                                      InLoadOrderLinks);
 362
 363         /* Copy the entry */
 364         LdrCoreEntries[i] = *LdrEntry;
 365
 366         /* Insert the copy into the list */
 367         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
 368     }
 369 }
 370
 371 VOID
 372 NTAPI
 373 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 374 {
 375     CCHAR Cpu;
 376     PKTHREAD InitialThread;
 377     ULONG64 InitialStack;
 378     PKIPCR Pcr;
 379
 380     /* HACK */
 381     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
 382     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
 383
 384     /* Save the loader block */
 385     KeLoaderBlock = LoaderBlock;
 386
 387     /* Get the current CPU number */
 388     Cpu = (CCHAR)KeNumberProcessors++; // FIXME
 389
 390     /* LoaderBlock initialization for Cpu 0 */
 391     if (Cpu == 0)
 392     {
 393         /* Set the initial stack, idle thread and process */
 394         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 395         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 396         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
 397         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
 398     }
 399
 400     /* Get Pcr from loader block */
 401     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
 402
 403     /* Set the PRCB for this Processor */
 404     KiProcessorBlock[Cpu] = &Pcr->Prcb;
 405
 406     /* Align stack to 16 bytes */
 407     LoaderBlock->KernelStack &= ~(16 - 1);
 408
 409     /* Save the initial thread and stack */
 410     InitialStack = LoaderBlock->KernelStack; // Checkme
 411     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 412
 413     /* Set us as the current process */
 414     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
 415
 416     /* Initialize the PCR */
 417     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
 418
 419     /* Initialize the CPU features */
 420     KiInitializeCpu(Pcr);
 421
 422     /* Initial setup for the boot CPU */
 423     if (Cpu == 0)
 424     {
 425         /* Initialize the module list (ntos, hal, kdcom) */
 426         KiInitModuleList(LoaderBlock);
 427
 428         /* Setup the TSS descriptors and entries */
 429         KiInitializeTss(Pcr->TssBase, InitialStack);
 430
 431         /* Setup the IDT */
 432         KeInitExceptions();
 433
 434          /* Initialize debugging system */
 435         KdInitSystem(0, KeLoaderBlock);
 436
 437         /* Check for break-in */
 438         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 439     }
 440
 441     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
 442            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
 443
 444     /* Acquire lock */
 445     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
 446     {
 447         /* Loop until lock is free */
 448         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
 449     }
 450
 451     /* Initialize the Processor with HAL */
 452     HalInitializeProcessor(Cpu, KeLoaderBlock);
 453
 454     /* Set processor as active */
 455     KeActiveProcessors |= 1ULL << Cpu;
 456
 457     /* Release lock */
 458     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
 459
 460     /* Raise to HIGH_LEVEL */
 461     KfRaiseIrql(HIGH_LEVEL);
 462
 463     /* Machine specific kernel initialization */
 464     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
 465
 466     /* Switch to new kernel stack and start kernel bootstrapping */
 467     KiSwitchToBootStack(InitialStack & ~3);
 468 }
 469