ntoskrnl/include/internal/amd64/ke.h

   1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
   2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
   3
   4 #ifdef __cplusplus
   5 extern "C" {
   6 #endif
   7
   8 #define X86_EFLAGS_TF           0x00000100 /* Trap flag */
   9 #define X86_EFLAGS_IF           0x00000200 /* Interrupt Enable flag */
  10 #define X86_EFLAGS_IOPL         0x00003000 /* I/O Privilege Level bits */
  11 #define X86_EFLAGS_NT           0x00004000 /* Nested Task flag */
  12 #define X86_EFLAGS_RF           0x00010000 /* Resume flag */
  13 #define X86_EFLAGS_VM           0x00020000 /* Virtual Mode */
  14 #define X86_EFLAGS_ID           0x00200000 /* CPUID detection flag */
  15
  16 #define X86_CR0_PE              0x00000001 /* enable Protected Mode */
  17 #define X86_CR0_NE              0x00000020 /* enable native FPU error reporting */
  18 #define X86_CR0_TS              0x00000008 /* enable exception on FPU instruction for task switch */
  19 #define X86_CR0_EM              0x00000004 /* enable FPU emulation (disable FPU) */
  20 #define X86_CR0_MP              0x00000002 /* enable FPU monitoring */
  21 #define X86_CR0_WP              0x00010000 /* enable Write Protect (copy on write) */
  22 #define X86_CR0_PG              0x80000000 /* enable Paging */
  23
  24 #define X86_CR4_PAE             0x00000020 /* enable physical address extensions */
  25 #define X86_CR4_PGE             0x00000080 /* enable global pages */
  26 #define X86_CR4_OSFXSR          0x00000200 /* enable FXSAVE/FXRSTOR instructions */
  27 #define X86_CR4_OSXMMEXCPT      0x00000400 /* enable #XF exception */
  28
  29 /* EDX flags */
  30 #define X86_FEATURE_FPU         0x00000001 /* x87 FPU is present */
  31 #define X86_FEATURE_VME         0x00000002 /* Virtual 8086 Extensions are present */
  32 #define X86_FEATURE_DBG         0x00000004 /* Debugging extensions are present */
  33 #define X86_FEATURE_PSE         0x00000008 /* Page Size Extension is present */
  34 #define X86_FEATURE_TSC         0x00000010 /* time stamp counters are present */
  35 #define X86_FEATURE_PAE         0x00000040 /* physical address extension is present */
  36 #define X86_FEATURE_CX8         0x00000100 /* CMPXCHG8B instruction present */
  37 #define X86_FEATURE_SYSCALL     0x00000800 /* SYSCALL/SYSRET support present */
  38 #define X86_FEATURE_MTTR        0x00001000 /* Memory type range registers are present */
  39 #define X86_FEATURE_PGE         0x00002000 /* Page Global Enable */
  40 #define X86_FEATURE_CMOV        0x00008000 /* "Conditional move" instruction supported */
  41 #define X86_FEATURE_PAT         0x00010000 /* Page Attribute Table is supported */
  42 #define X86_FEATURE_DS          0x00200000 /* Debug Store is present */
  43 #define X86_FEATURE_MMX         0x00800000 /* MMX extension present */
  44 #define X86_FEATURE_FXSR        0x01000000 /* FXSAVE/FXRSTOR instructions present */
  45 #define X86_FEATURE_SSE         0x02000000 /* SSE extension present */
  46 #define X86_FEATURE_SSE2        0x04000000 /* SSE2 extension present */
  47 #define X86_FEATURE_HT          0x10000000 /* Hyper-Threading present */
  48
  49 /* ECX flags */
  50 #define X86_FEATURE_SSE3        0x00000001 /* SSE3 is supported */
  51 #define X86_FEATURE_MONITOR     0x00000008 /* SSE3 Monitor instructions supported */
  52 #define X86_FEATURE_VMX         0x00000020 /* Virtual Machine eXtensions are available */
  53 #define X86_FEATURE_SSSE3       0x00000200 /* Supplemental SSE3 are available */
  54 #define X86_FEATURE_FMA3        0x00001000 /* Fused multiple-add supported */
  55 #define X86_FEATURE_CX16        0x00002000 /* CMPXCHG16B instruction are available */
  56 #define X86_FEATURE_PCID        0x00020000 /* Process Context IDentifiers are supported */
  57 #define X86_FEATURE_SSE41       0x00080000 /* SSE 4.1 is supported */
  58 #define X86_FEATURE_SSE42       0x00100000 /* SSE 4.2 is supported */
  59 #define X86_FEATURE_POPCNT      0x00800000 /* POPCNT instruction is available */
  60 #define X86_FEATURE_XSAVE       0x04000000 /* XSAVE family are available */
  61
  62 /* EDX extended flags */
  63 #define X86_FEATURE_NX          0x00100000 /* NX support present */
  64
  65 #define X86_EXT_FEATURE_SSE3    0x00000001 /* SSE3 extension present */
  66 #define X86_EXT_FEATURE_3DNOW   0x40000000 /* 3DNOW! extension present */
  67
  68 #define FRAME_EDITED        0xFFF8
  69
  70 #define X86_MSR_GSBASE          0xC0000101
  71 #define X86_MSR_KERNEL_GSBASE   0xC0000102
  72 #define X86_MSR_EFER            0xC0000080
  73 #define X86_MSR_STAR            0xC0000081
  74 #define X86_MSR_LSTAR           0xC0000082
  75 #define X86_MSR_CSTAR           0xC0000083
  76 #define X86_MSR_SFMASK          0xC0000084
  77
  78 #define EFER_SCE    0x0001
  79 #define EFER_LME    0x0100
  80 #define EFER_LMA    0x0400
  81 #define EFER_NXE    0x0800
  82 #define EFER_SVME   0x1000
  83 #define EFER_FFXSR  0x4000
  84
  85 #define AMD64_TSS 9
  86
  87 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL
  88
  89 #ifndef __ASM__
  90
  91 #include "intrin_i.h"
  92
  93 typedef struct _KIDT_INIT
  94 {
  95     UCHAR InterruptId;
  96     UCHAR Dpl;
  97     UCHAR IstIndex;
  98     PVOID ServiceRoutine;
  99 } KIDT_INIT, *PKIDT_INIT;
 100
 101 #include <pshpack1.h>
 102 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY
 103 {
 104     UCHAR _Op_nop;
 105     UCHAR _Op_push;
 106     UCHAR _Vector;
 107     UCHAR _Op_jmp;
 108     ULONG RelativeAddress;
 109 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY;
 110 #include <poppack.h>
 111
 112 extern ULONG KeI386NpxPresent;
 113 extern ULONG KeI386XMMIPresent;
 114 extern ULONG KeI386FxsrPresent;
 115 extern ULONG KeI386CpuType;
 116 extern ULONG KeI386CpuStep;
 117
 118 //
 119 // INT3 is 1 byte long
 120 //
 121 #define KD_BREAKPOINT_TYPE        UCHAR
 122 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
 123 #define KD_BREAKPOINT_VALUE       0xCC
 124
 125 //
 126 // One-liners for getting and setting special purpose registers in portable code
 127 //
 128 FORCEINLINE
 129 ULONG_PTR
 130 KeGetContextPc(PCONTEXT Context)
 131 {
 132     return Context->Rip;
 133 }
 134
 135 FORCEINLINE
 136 VOID
 137 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
 138 {
 139     Context->Rip = ProgramCounter;
 140 }
 141
 142 FORCEINLINE
 143 ULONG_PTR
 144 KeGetContextReturnRegister(PCONTEXT Context)
 145 {
 146     return Context->Rax;
 147 }
 148
 149 FORCEINLINE
 150 VOID
 151 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
 152 {
 153     Context->Rax = ReturnValue;
 154 }
 155
 156 FORCEINLINE
 157 ULONG_PTR
 158 KeGetContextStackRegister(PCONTEXT Context)
 159 {
 160     return Context->Rsp;
 161 }
 162
 163 FORCEINLINE
 164 ULONG_PTR
 165 KeGetContextFrameRegister(PCONTEXT Context)
 166 {
 167     return Context->Rbp;
 168 }
 169
 170 FORCEINLINE
 171 VOID
 172 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
 173 {
 174     Context->Rbp = Frame;
 175 }
 176
 177 FORCEINLINE
 178 ULONG_PTR
 179 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
 180 {
 181     return TrapFrame->Rip;
 182 }
 183
 184 FORCEINLINE
 185 PKTRAP_FRAME
 186 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
 187 {
 188     return (PKTRAP_FRAME)TrapFrame->TrapFrame;
 189 }
 190
 191 FORCEINLINE
 192 ULONG_PTR
 193 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
 194 {
 195     return TrapFrame->Rsp;
 196 }
 197
 198 FORCEINLINE
 199 ULONG_PTR
 200 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
 201 {
 202     return TrapFrame->Rbp;
 203 }
 204
 205 //
 206 // Macro to get trap and exception frame from a thread stack
 207 //
 208 #define KeGetTrapFrame(Thread) \
 209     ((PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
 210                    sizeof(KTRAP_FRAME)))
 211
 212 //
 213 // Macro to get context switches from the PRCB
 214 // All architectures but x86 have it in the PRCB's KeContextSwitches
 215 //
 216 #define KeGetContextSwitches(Prcb)  \
 217     (Prcb->KeContextSwitches)
 218
 219 //
 220 // Macro to get the second level cache size field name which differs between
 221 // CISC and RISC architectures, as the former has unified I/D cache
 222 //
 223 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
 224
 225 #define KeGetExceptionFrame(Thread) \
 226     (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \
 227                         sizeof(KEXCEPTION_FRAME))
 228
 229 //
 230 // Returns the Interrupt State from a Trap Frame.
 231 // ON = TRUE, OFF = FALSE
 232 //
 233 #define KeGetTrapFrameInterruptState(TrapFrame) \
 234         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
 235
 236 /* Diable interrupts and return whether they were enabled before */
 237 FORCEINLINE
 238 BOOLEAN
 239 KeDisableInterrupts(VOID)
 240 {
 241     ULONG_PTR Flags;
 242
 243     /* Get EFLAGS and check if the interrupt bit is set */
 244     Flags = __readeflags();
 245
 246     /* Disable interrupts */
 247     _disable();
 248     return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE;
 249 }
 250
 251 /* Restore previous interrupt state */
 252 FORCEINLINE
 253 VOID
 254 KeRestoreInterrupts(BOOLEAN WereEnabled)
 255 {
 256     if (WereEnabled) _enable();
 257 }
 258
 259 //
 260 // Invalidates the TLB entry for a specified address
 261 //
 262 FORCEINLINE
 263 VOID
 264 KeInvalidateTlbEntry(IN PVOID Address)
 265 {
 266     /* Invalidate the TLB entry for this address */
 267     __invlpg(Address);
 268 }
 269
 270 FORCEINLINE
 271 VOID
 272 KeFlushProcessTb(VOID)
 273 {
 274     /* Flush the TLB by resetting CR3 */
 275     __writecr3(__readcr3());
 276 }
 277
 278 FORCEINLINE
 279 VOID
 280 KeSweepICache(IN PVOID BaseAddress,
 281               IN SIZE_T FlushSize)
 282 {
 283     //
 284     // Always sweep the whole cache
 285     //
 286     UNREFERENCED_PARAMETER(BaseAddress);
 287     UNREFERENCED_PARAMETER(FlushSize);
 288     __wbinvd();
 289 }
 290
 291 FORCEINLINE
 292 VOID
 293 KiRundownThread(IN PKTHREAD Thread)
 294 {
 295     /* Nothing to do */
 296 }
 297
 298 /* Registers an interrupt handler with an IDT vector */
 299 FORCEINLINE
 300 VOID
 301 KeRegisterInterruptHandler(IN ULONG Vector,
 302                            IN PVOID Handler)
 303 {
 304     UCHAR Entry;
 305     PKIDTENTRY64 Idt;
 306
 307     /* Get the entry from the HAL */
 308     Entry = HalVectorToIDTEntry(Vector);
 309
 310     /* Now set the data */
 311     Idt = &KeGetPcr()->IdtBase[Entry];
 312     Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff;
 313     Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff;
 314     Idt->OffsetHigh = (ULONG_PTR)Handler >> 32;
 315     Idt->Selector = KGDT64_R0_CODE;
 316     Idt->IstIndex = 0;
 317     Idt->Type = 0x0e;
 318     Idt->Dpl = 0;
 319     Idt->Present = 1;
 320     Idt->Reserved0 = 0;
 321     Idt->Reserved1 = 0;
 322 }
 323
 324 /* Returns the registered interrupt handler for a given IDT vector */
 325 FORCEINLINE
 326 PVOID
 327 KeQueryInterruptHandler(IN ULONG Vector)
 328 {
 329     UCHAR Entry;
 330     PKIDTENTRY64 Idt;
 331
 332     /* Get the entry from the HAL */
 333     Entry = HalVectorToIDTEntry(Vector);
 334
 335     /* Get the IDT entry */
 336     Idt = &KeGetPcr()->IdtBase[Entry];
 337
 338     /* Return the address */
 339     return (PVOID)((ULONG64)Idt->OffsetHigh << 32 |
 340                    (ULONG64)Idt->OffsetMiddle << 16 |
 341                    (ULONG64)Idt->OffsetLow);
 342 }
 343
 344 FORCEINLINE
 345 VOID
 346 KiSendEOI(VOID)
 347 {
 348     /* Write 0 to the apic EOI register */
 349     *((volatile ULONG*)APIC_EOI_REGISTER) = 0;
 350 }
 351
 352 FORCEINLINE
 353 VOID
 354 KiEndInterrupt(IN KIRQL Irql,
 355                IN PKTRAP_FRAME TrapFrame)
 356 {
 357     /* Make sure this is from the clock handler */
 358     ASSERT(TrapFrame->ErrorCode == 0xc10c4);
 359     //KeLowerIrql(Irql);
 360 }
 361
 362 FORCEINLINE
 363 BOOLEAN
 364 KiUserTrap(IN PKTRAP_FRAME TrapFrame)
 365 {
 366     /* Anything else but Ring 0 is Ring 3 */
 367     return !!(TrapFrame->SegCs & MODE_MASK);
 368 }
 369
 370 //
 371 // PERF Code
 372 //
 373 FORCEINLINE
 374 VOID
 375 Ki386PerfEnd(VOID)
 376 {
 377     extern ULONGLONG BootCyclesEnd, BootCycles;
 378     BootCyclesEnd = __rdtsc();
 379     DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
 380     DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
 381              KeGetCurrentPrcb()->InterruptCount,
 382              KeGetCurrentPrcb()->KeSystemCalls,
 383              KeGetContextSwitches(KeGetCurrentPrcb()));
 384 }
 385
 386 struct _KPCR;
 387
 388 DECLSPEC_NORETURN VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack);
 389 VOID KiDivideErrorFault(VOID);
 390 VOID KiDebugTrapOrFault(VOID);
 391 VOID KiNmiInterrupt(VOID);
 392 VOID KiBreakpointTrap(VOID);
 393 VOID KiOverflowTrap(VOID);
 394 VOID KiBoundFault(VOID);
 395 VOID KiInvalidOpcodeFault(VOID);
 396 VOID KiNpxNotAvailableFault(VOID);
 397 VOID KiDoubleFaultAbort(VOID);
 398 VOID KiNpxSegmentOverrunAbort(VOID);
 399 VOID KiInvalidTssFault(VOID);
 400 VOID KiSegmentNotPresentFault(VOID);
 401 VOID KiStackFault(VOID);
 402 VOID KiGeneralProtectionFault(VOID);
 403 VOID KiPageFault(VOID);
 404 VOID KiFloatingErrorFault(VOID);
 405 VOID KiAlignmentFault(VOID);
 406 VOID KiMcheckAbort(VOID);
 407 VOID KiXmmException(VOID);
 408 VOID KiApcInterrupt(VOID);
 409 VOID KiRaiseAssertion(VOID);
 410 VOID KiDebugServiceTrap(VOID);
 411 VOID KiDpcInterrupt(VOID);
 412 VOID KiIpiInterrupt(VOID);
 413
 414 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id);
 415 VOID Ki386InitializeLdt(VOID);
 416 VOID Ki386SetProcessorFeatures(VOID);
 417 VOID KiGetCacheInformation(VOID);
 418 VOID KiSetProcessorType(VOID);
 419 ULONG64 KiGetFeatureBits(VOID);
 420 VOID KiInitializeCpuFeatures(VOID);
 421 #if DBG
 422 VOID KiReportCpuFeatures(IN PKPRCB Prcb);
 423 #endif
 424
 425 ULONG KeAllocateGdtSelector(ULONG Desc[2]);
 426 VOID KeFreeGdtSelector(ULONG Entry);
 427 VOID NtEarlyInitVdm(VOID);
 428 VOID KeApplicationProcessorInitDispatcher(VOID);
 429 VOID KeCreateApplicationProcessorIdleThread(ULONG Id);
 430
 431 VOID
 432 Ke386InitThreadWithContext(PKTHREAD Thread,
 433                            PKSYSTEM_ROUTINE SystemRoutine,
 434                            PKSTART_ROUTINE StartRoutine,
 435                            PVOID StartContext,
 436                            PCONTEXT Context);
 437 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \
 438   Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context)
 439
 440 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */
 441 VOID
 442 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine,
 443                 PKSTART_ROUTINE StartRoutine,
 444                 PVOID StartContext,
 445                 BOOLEAN UserThread,
 446                 KTRAP_FRAME TrapFrame);
 447 #endif
 448
 449 #endif /* __ASM__ */
 450
 451 // HACK
 452 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount;
 453
 454 // win64 uses DMA macros, this one is not defined
 455 NTHALAPI
 456 NTSTATUS
 457 NTAPI
 458 HalAllocateAdapterChannel(
 459   IN PADAPTER_OBJECT  AdapterObject,
 460   IN PWAIT_CONTEXT_BLOCK  Wcb,
 461   IN ULONG  NumberOfMapRegisters,
 462   IN PDRIVER_CONTROL  ExecutionRoutine);
 463
 464 FORCEINLINE
 465 PULONG_PTR
 466 KiGetUserModeStackAddress(void)
 467 {
 468     return &PsGetCurrentThread()->Tcb.TrapFrame->Rsp;
 469 }
 470
 471 VOID
 472 KiSetTrapContext(
 473     _Out_ PKTRAP_FRAME TrapFrame,
 474     _In_ PCONTEXT Context,
 475     _In_ KPROCESSOR_MODE RequestorMode);
 476
 477 // Exits to user mode, only restores the trap frame, zeroes the non-volatile registers
 478 DECLSPEC_NORETURN
 479 VOID
 480 KiUserCallbackExit(
 481     _In_ PKTRAP_FRAME TrapFrame);
 482
 483 #ifdef __cplusplus
 484 } // extern "C"
 485 #endif
 486
 487 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */
 488
 489 /* EOF */