ntoskrnl/ke/amd64/kiinit.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/amd64/kiinit.c
   5  * PURPOSE:         Kernel Initialization for x86 CPUs
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Timo Kreuzer (timo.kreuzer@reactos.org)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #define REQUIRED_FEATURE_BITS (KF_RDTSC|KF_CR4|KF_CMPXCHG8B|KF_XMMI|KF_XMMI64| \
  17                                KF_LARGE_PAGE|KF_FAST_SYSCALL|KF_GLOBAL_PAGE| \
  18                                KF_CMOV|KF_PAT|KF_MMX|KF_FXSR|KF_NX_BIT|KF_MTRR)
  19
  20 /* GLOBALS *******************************************************************/
  21
  22 /* Function pointer for early debug prints */
  23 ULONG (*FrLdrDbgPrint)(const char *Format, ...);
  24
  25 /* Spinlocks used only on X86 */
  26 KSPIN_LOCK KiFreezeExecutionLock;
  27
  28
  29 KIPCR KiInitialPcr;
  30
  31 /* Boot and double-fault/NMI/DPC stack */
  32 UCHAR DECLSPEC_ALIGN(16) P0BootStackData[KERNEL_STACK_SIZE] = {0};
  33 UCHAR DECLSPEC_ALIGN(16) KiDoubleFaultStackData[KERNEL_STACK_SIZE] = {0};
  34 ULONG_PTR P0BootStack = (ULONG_PTR)&P0BootStackData[KERNEL_STACK_SIZE];
  35 ULONG_PTR KiDoubleFaultStack = (ULONG_PTR)&KiDoubleFaultStackData[KERNEL_STACK_SIZE];
  36
  37 void KiInitializeSegments();
  38 void KiSystemCallEntry64();
  39 void KiSystemCallEntry32();
  40
  41 /* FUNCTIONS *****************************************************************/
  42
  43 CODE_SEG("INIT")
  44 VOID
  45 NTAPI
  46 KiInitMachineDependent(VOID)
  47 {
  48     /* Check for large page support */
  49     if (KeFeatureBits & KF_LARGE_PAGE)
  50     {
  51         /* FIXME: Support this */
  52         DPRINT("Large Page support detected but not yet taken advantage of!\n");
  53     }
  54
  55     /* Check for global page support */
  56     if (KeFeatureBits & KF_GLOBAL_PAGE)
  57     {
  58         /* FIXME: Support this */
  59         DPRINT("Global Page support detected but not yet taken advantage of!\n");
  60     }
  61
  62     /* Check if we have MTRR */
  63     if (KeFeatureBits & KF_MTRR)
  64     {
  65         /* FIXME: Support this */
  66         DPRINT("MTRR support detected but not yet taken advantage of!\n");
  67     }
  68
  69     /* Check for PAT and/or MTRR support */
  70     if (KeFeatureBits & KF_PAT)
  71     {
  72         /* FIXME: Support this */
  73         DPRINT("PAT support detected but not yet taken advantage of!\n");
  74     }
  75
  76 //        /* Allocate the IOPM save area */
  77 //        Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
  78 //                                                  IOPM_SIZE,
  79 //                                                  '  eK');
  80 //        if (!Ki386IopmSaveArea)
  81 //        {
  82 //            /* Bugcheck. We need this for V86/VDM support. */
  83 //            KeBugCheckEx(NO_PAGES_AVAILABLE, 2, IOPM_SIZE, 0, 0);
  84 //        }
  85
  86 }
  87
  88 VOID
  89 NTAPI
  90 KiInitializePcr(IN PKIPCR Pcr,
  91                 IN ULONG ProcessorNumber,
  92                 IN PKTHREAD IdleThread,
  93                 IN PVOID DpcStack)
  94 {
  95     KDESCRIPTOR GdtDescriptor = {{0},0,0}, IdtDescriptor = {{0},0,0};
  96     PKGDTENTRY64 TssEntry;
  97     USHORT Tr = 0;
  98
  99     /* Zero out the PCR */
 100     RtlZeroMemory(Pcr, sizeof(KIPCR));
 101
 102     /* Set pointers to ourselves */
 103     Pcr->Self = (PKPCR)Pcr;
 104     Pcr->CurrentPrcb = &Pcr->Prcb;
 105
 106     /* Set the PCR Version */
 107     Pcr->MajorVersion = PCR_MAJOR_VERSION;
 108     Pcr->MinorVersion = PCR_MINOR_VERSION;
 109
 110     /* Set the PRCB Version */
 111     Pcr->Prcb.MajorVersion = 1;
 112     Pcr->Prcb.MinorVersion = 1;
 113
 114     /* Set the Build Type */
 115     Pcr->Prcb.BuildType = 0;
 116 #ifndef CONFIG_SMP
 117     Pcr->Prcb.BuildType |= PRCB_BUILD_UNIPROCESSOR;
 118 #endif
 119 #if DBG
 120     Pcr->Prcb.BuildType |= PRCB_BUILD_DEBUG;
 121 #endif
 122
 123     /* Set the Processor Number and current Processor Mask */
 124     Pcr->Prcb.Number = (UCHAR)ProcessorNumber;
 125     Pcr->Prcb.SetMember = 1ULL << ProcessorNumber;
 126
 127     /* Get GDT and IDT descriptors */
 128     __sgdt(&GdtDescriptor.Limit);
 129     __sidt(&IdtDescriptor.Limit);
 130     Pcr->GdtBase = (PVOID)GdtDescriptor.Base;
 131     Pcr->IdtBase = (PKIDTENTRY)IdtDescriptor.Base;
 132
 133     /* Get TSS Selector */
 134     __str(&Tr);
 135     ASSERT(Tr == KGDT64_SYS_TSS);
 136
 137     /* Get TSS Entry */
 138     TssEntry = KiGetGdtEntry(Pcr->GdtBase, Tr);
 139
 140     /* Get the KTSS itself */
 141     Pcr->TssBase = KiGetGdtDescriptorBase(TssEntry);
 142
 143     Pcr->Prcb.RspBase = Pcr->TssBase->Rsp0; // FIXME
 144
 145     /* Set DPC Stack */
 146     Pcr->Prcb.DpcStack = DpcStack;
 147
 148     /* Setup the processor set */
 149     Pcr->Prcb.MultiThreadProcessorSet = Pcr->Prcb.SetMember;
 150
 151     /* Clear DR6/7 to cleanup bootloader debugging */
 152     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr6 = 0;
 153     Pcr->Prcb.ProcessorState.SpecialRegisters.KernelDr7 = 0;
 154
 155     /* Set the Current Thread */
 156     Pcr->Prcb.CurrentThread = IdleThread;
 157
 158     /* Start us out at PASSIVE_LEVEL */
 159     Pcr->Irql = PASSIVE_LEVEL;
 160     KeSetCurrentIrql(PASSIVE_LEVEL);
 161 }
 162
 163 CODE_SEG("INIT")
 164 VOID
 165 NTAPI
 166 KiInitializeCpu(PKIPCR Pcr)
 167 {
 168     ULONG64 Pat;
 169     ULONG FeatureBits;
 170
 171     /* Initialize gs */
 172     KiInitializeSegments();
 173
 174     /* Set GS base */
 175     __writemsr(MSR_GS_BASE, (ULONG64)Pcr);
 176     __writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
 177
 178     /* Detect and set the CPU Type */
 179     KiSetProcessorType();
 180
 181     /* Get the processor features for this CPU */
 182     FeatureBits = KiGetFeatureBits();
 183
 184     /* Check if we support all needed features */
 185     if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
 186     {
 187         /* If not, bugcheck system */
 188         FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
 189                 FeatureBits, REQUIRED_FEATURE_BITS);
 190         KeBugCheck(0);
 191     }
 192
 193     /* Set DEP to always on */
 194     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 195     FeatureBits |= KF_NX_ENABLED;
 196
 197     /* Save feature bits */
 198     Pcr->Prcb.FeatureBits = FeatureBits;
 199
 200     /* Enable fx save restore support */
 201     __writecr4(__readcr4() | CR4_FXSR);
 202
 203     /* Enable XMMI exceptions */
 204     __writecr4(__readcr4() | CR4_XMMEXCPT);
 205
 206     /* Enable Write-Protection */
 207     __writecr0(__readcr0() | CR0_WP);
 208
 209     /* Disable fpu monitoring */
 210     __writecr0(__readcr0() & ~CR0_MP);
 211
 212     /* Disable x87 fpu exceptions */
 213     __writecr0(__readcr0() & ~CR0_NE);
 214
 215     /* LDT is unused */
 216     __lldt(0);
 217
 218     /* Set the systemcall entry points */
 219     __writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
 220     __writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
 221
 222     __writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
 223                          ((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
 224
 225     /* Set the flags to be cleared when doing a syscall */
 226     __writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
 227
 228     /* Enable syscall instruction and no-execute support */
 229     __writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
 230
 231     /* Initialize the PAT */
 232     Pat = (PAT_WB << 0)  | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
 233           (PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
 234     __writemsr(MSR_PAT, Pat);
 235 }
 236
 237 VOID
 238 FASTCALL
 239 KiInitializeTss(IN PKTSS64 Tss,
 240                 IN UINT64 Stack)
 241 {
 242     PKGDTENTRY64 TssEntry;
 243
 244     /* Get pointer to the GDT entry */
 245     TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
 246
 247     /* Initialize the GDT entry */
 248     KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
 249
 250     /* Zero out the TSS */
 251     RtlZeroMemory(Tss, sizeof(KTSS64));
 252
 253     /* FIXME: I/O Map? */
 254     Tss->IoMapBase = 0x68;
 255
 256     /* Setup ring 0 stack pointer */
 257     Tss->Rsp0 = Stack;
 258
 259     /* Setup a stack for Double Fault Traps */
 260     Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
 261
 262     /* Setup a stack for CheckAbort Traps */
 263     Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
 264
 265     /* Setup a stack for NMI Traps */
 266     Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
 267
 268     /* Load the task register */
 269     __ltr(KGDT64_SYS_TSS);
 270 }
 271
 272 CODE_SEG("INIT")
 273 VOID
 274 NTAPI
 275 KiInitializeKernelMachineDependent(
 276     IN PKPRCB Prcb,
 277     IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 278 {
 279     /* Set boot-level flags */
 280     KeI386CpuType = Prcb->CpuType;
 281     KeI386CpuStep = Prcb->CpuStep;
 282     KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_AMD64;
 283     KeProcessorLevel = (USHORT)Prcb->CpuType;
 284     if (Prcb->CpuID)
 285         KeProcessorRevision = Prcb->CpuStep;
 286
 287     /* Set basic CPU Features that user mode can read */
 288     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] = TRUE;
 289     SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] = TRUE;
 290     SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = TRUE;
 291     SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = TRUE; // ???
 292     SharedUserData->ProcessorFeatures[PF_NX_ENABLED] = TRUE;
 293     SharedUserData->ProcessorFeatures[PF_FASTFAIL_AVAILABLE] = TRUE;
 294     SharedUserData->ProcessorFeatures[PF_XSAVE_ENABLED] = TRUE;
 295     SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
 296         (Prcb->FeatureBits & KF_MMX) ? TRUE: FALSE;
 297     SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
 298         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI)) ? TRUE: FALSE;
 299     SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
 300         ((Prcb->FeatureBits & KF_FXSR) && (Prcb->FeatureBits & KF_XMMI64)) ? TRUE: FALSE;
 301     SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
 302         (Prcb->FeatureBits & KF_3DNOW) ? TRUE: FALSE;
 303     SharedUserData->ProcessorFeatures[PF_SSE3_INSTRUCTIONS_AVAILABLE] =
 304         (Prcb->FeatureBits & KF_SSE3) ? TRUE: FALSE;
 305     SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE128] =
 306         (Prcb->FeatureBits & KF_CMPXCHG16B) ? TRUE: FALSE;
 307
 308     /* Set the default NX policy (opt-in) */
 309     SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
 310
 311     /* Check if NPX is always on */
 312     if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
 313     {
 314         /* Set it always on */
 315         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
 316         Prcb->FeatureBits |= KF_NX_ENABLED;
 317     }
 318     else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
 319     {
 320         /* Set it in opt-out mode */
 321         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
 322         Prcb->FeatureBits |= KF_NX_ENABLED;
 323     }
 324     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
 325              (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
 326     {
 327         /* Set the feature bits */
 328         Prcb->FeatureBits |= KF_NX_ENABLED;
 329     }
 330     else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
 331              (strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
 332     {
 333         /* Set disabled mode */
 334         SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
 335         Prcb->FeatureBits |= KF_NX_DISABLED;
 336     }
 337 }
 338
 339 static LDR_DATA_TABLE_ENTRY LdrCoreEntries[3];
 340
 341 void
 342 KiInitModuleList(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 343 {
 344     PLDR_DATA_TABLE_ENTRY LdrEntry;
 345     PLIST_ENTRY Entry;
 346     ULONG i;
 347
 348     /* Initialize the list head */
 349     InitializeListHead(&PsLoadedModuleList);
 350
 351     /* Loop the first 3 entries */
 352     for (Entry = LoaderBlock->LoadOrderListHead.Flink, i = 0;
 353          Entry != &LoaderBlock->LoadOrderListHead && i < 3;
 354          Entry = Entry->Flink, i++)
 355     {
 356         /* Get the data table entry */
 357         LdrEntry = CONTAINING_RECORD(Entry,
 358                                      LDR_DATA_TABLE_ENTRY,
 359                                      InLoadOrderLinks);
 360
 361         /* Copy the entry */
 362         LdrCoreEntries[i] = *LdrEntry;
 363
 364         /* Insert the copy into the list */
 365         InsertTailList(&PsLoadedModuleList, &LdrCoreEntries[i].InLoadOrderLinks);
 366     }
 367 }
 368
 369 CODE_SEG("INIT")
 370 VOID
 371 NTAPI
 372 KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 373 {
 374     CCHAR Cpu;
 375     PKTHREAD InitialThread;
 376     ULONG64 InitialStack;
 377     PKIPCR Pcr;
 378
 379     /* HACK */
 380     FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
 381     //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
 382
 383     /* Save the loader block */
 384     KeLoaderBlock = LoaderBlock;
 385
 386     /* Get the current CPU number */
 387     Cpu = KeNumberProcessors++; // FIXME
 388
 389     /* LoaderBlock initialization for Cpu 0 */
 390     if (Cpu == 0)
 391     {
 392         /* Set the initial stack, idle thread and process */
 393         LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
 394         LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
 395         LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
 396         LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
 397     }
 398
 399     /* Get Pcr from loader block */
 400     Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
 401
 402     /* Set the PRCB for this Processor */
 403     KiProcessorBlock[Cpu] = &Pcr->Prcb;
 404
 405     /* Align stack to 16 bytes */
 406     LoaderBlock->KernelStack &= ~(16 - 1);
 407
 408     /* Save the initial thread and stack */
 409     InitialStack = LoaderBlock->KernelStack; // Checkme
 410     InitialThread = (PKTHREAD)LoaderBlock->Thread;
 411
 412     /* Set us as the current process */
 413     InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
 414
 415     /* Initialize the PCR */
 416     KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
 417
 418     /* Initialize the CPU features */
 419     KiInitializeCpu(Pcr);
 420
 421     /* Initial setup for the boot CPU */
 422     if (Cpu == 0)
 423     {
 424         /* Initialize the module list (ntos, hal, kdcom) */
 425         KiInitModuleList(LoaderBlock);
 426
 427         /* Setup the TSS descriptors and entries */
 428         KiInitializeTss(Pcr->TssBase, InitialStack);
 429
 430         /* Setup the IDT */
 431         KeInitExceptions();
 432
 433          /* Initialize debugging system */
 434         KdInitSystem(0, KeLoaderBlock);
 435
 436         /* Check for break-in */
 437         if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
 438     }
 439
 440     DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
 441            Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
 442
 443     /* Acquire lock */
 444     while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
 445     {
 446         /* Loop until lock is free */
 447         while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
 448     }
 449
 450     /* Initialize the Processor with HAL */
 451     HalInitializeProcessor(Cpu, KeLoaderBlock);
 452
 453     /* Set processor as active */
 454     KeActiveProcessors |= 1ULL << Cpu;
 455
 456     /* Release lock */
 457     InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
 458
 459     /* Raise to HIGH_LEVEL */
 460     KfRaiseIrql(HIGH_LEVEL);
 461
 462     /* Machine specific kernel initialization */
 463     if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
 464
 465     /* Switch to new kernel stack and start kernel bootstrapping */
 466     KiSwitchToBootStack(InitialStack & ~3);
 467 }
 468