ntoskrnl/include/internal/amd64/ke.h

   1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
   2 #define __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H
   3
   4 #define X86_EFLAGS_TF           0x00000100 /* Trap flag */
   5 #define X86_EFLAGS_IF           0x00000200 /* Interrupt Enable flag */
   6 #define X86_EFLAGS_IOPL         0x00003000 /* I/O Privilege Level bits */
   7 #define X86_EFLAGS_NT           0x00004000 /* Nested Task flag */
   8 #define X86_EFLAGS_RF           0x00010000 /* Resume flag */
   9 #define X86_EFLAGS_VM           0x00020000 /* Virtual Mode */
  10 #define X86_EFLAGS_ID           0x00200000 /* CPUID detection flag */
  11
  12 #define X86_CR0_PE              0x00000001 /* enable Protected Mode */
  13 #define X86_CR0_NE              0x00000020 /* enable native FPU error reporting */
  14 #define X86_CR0_TS              0x00000008 /* enable exception on FPU instruction for task switch */
  15 #define X86_CR0_EM              0x00000004 /* enable FPU emulation (disable FPU) */
  16 #define X86_CR0_MP              0x00000002 /* enable FPU monitoring */
  17 #define X86_CR0_WP              0x00010000 /* enable Write Protect (copy on write) */
  18 #define X86_CR0_PG              0x80000000 /* enable Paging */
  19
  20 #define X86_CR4_PAE             0x00000020 /* enable physical address extensions */
  21 #define X86_CR4_PGE             0x00000080 /* enable global pages */
  22 #define X86_CR4_OSFXSR          0x00000200 /* enable FXSAVE/FXRSTOR instructions */
  23 #define X86_CR4_OSXMMEXCPT      0x00000400 /* enable #XF exception */
  24
  25 /* EDX flags */
  26 #define X86_FEATURE_FPU         0x00000001 /* x87 FPU is present */
  27 #define X86_FEATURE_VME         0x00000002 /* Virtual 8086 Extensions are present */
  28 #define X86_FEATURE_DBG         0x00000004 /* Debugging extensions are present */
  29 #define X86_FEATURE_PSE         0x00000008 /* Page Size Extension is present */
  30 #define X86_FEATURE_TSC         0x00000010 /* time stamp counters are present */
  31 #define X86_FEATURE_PAE         0x00000040 /* physical address extension is present */
  32 #define X86_FEATURE_CX8         0x00000100 /* CMPXCHG8B instruction present */
  33 #define X86_FEATURE_SYSCALL     0x00000800 /* SYSCALL/SYSRET support present */
  34 #define X86_FEATURE_MTTR        0x00001000 /* Memory type range registers are present */
  35 #define X86_FEATURE_PGE         0x00002000 /* Page Global Enable */
  36 #define X86_FEATURE_CMOV        0x00008000 /* "Conditional move" instruction supported */
  37 #define X86_FEATURE_PAT         0x00010000 /* Page Attribute Table is supported */
  38 #define X86_FEATURE_DS          0x00200000 /* Debug Store is present */
  39 #define X86_FEATURE_MMX         0x00800000 /* MMX extension present */
  40 #define X86_FEATURE_FXSR        0x01000000 /* FXSAVE/FXRSTOR instructions present */
  41 #define X86_FEATURE_SSE         0x02000000 /* SSE extension present */
  42 #define X86_FEATURE_SSE2        0x04000000 /* SSE2 extension present */
  43 #define X86_FEATURE_HT          0x10000000 /* Hyper-Threading present */
  44
  45 /* ECX flags */
  46 #define X86_FEATURE_SSE3        0x00000001 /* SSE3 is supported */
  47 #define X86_FEATURE_MONITOR     0x00000008 /* SSE3 Monitor instructions supported */
  48 #define X86_FEATURE_VMX         0x00000020 /* Virtual Machine eXtensions are available */
  49 #define X86_FEATURE_SSSE3       0x00000200 /* Supplemental SSE3 are available */
  50 #define X86_FEATURE_FMA3        0x00001000 /* Fused multiple-add supported */
  51 #define X86_FEATURE_CX16        0x00002000 /* CMPXCHG16B instruction are available */
  52 #define X86_FEATURE_PCID        0x00020000 /* Process Context IDentifiers are supported */
  53 #define X86_FEATURE_SSE41       0x00080000 /* SSE 4.1 is supported */
  54 #define X86_FEATURE_SSE42       0x00100000 /* SSE 4.2 is supported */
  55 #define X86_FEATURE_POPCNT      0x00800000 /* POPCNT instruction is available */
  56 #define X86_FEATURE_XSAVE       0x04000000 /* XSAVE family are available */
  57
  58 /* EDX extended flags */
  59 #define X86_FEATURE_NX          0x00100000 /* NX support present */
  60
  61 #define X86_EXT_FEATURE_SSE3    0x00000001 /* SSE3 extension present */
  62 #define X86_EXT_FEATURE_3DNOW   0x40000000 /* 3DNOW! extension present */
  63
  64 #define FRAME_EDITED        0xFFF8
  65
  66 #define X86_MSR_GSBASE          0xC0000101
  67 #define X86_MSR_KERNEL_GSBASE   0xC0000102
  68 #define X86_MSR_EFER            0xC0000080
  69 #define X86_MSR_STAR            0xC0000081
  70 #define X86_MSR_LSTAR           0xC0000082
  71 #define X86_MSR_CSTAR           0xC0000083
  72 #define X86_MSR_SFMASK          0xC0000084
  73
  74 #define EFER_SCE    0x0001
  75 #define EFER_LME    0x0100
  76 #define EFER_LMA    0x0400
  77 #define EFER_NXE    0x0800
  78 #define EFER_SVME   0x1000
  79 #define EFER_FFXSR  0x4000
  80
  81 #define AMD64_TSS 9
  82
  83 #define APIC_EOI_REGISTER 0xFFFFFFFFFFFE00B0ULL
  84
  85 #ifndef __ASM__
  86
  87 #include "intrin_i.h"
  88
  89 typedef struct _KIDT_INIT
  90 {
  91     UCHAR InterruptId;
  92     UCHAR Dpl;
  93     UCHAR IstIndex;
  94     PVOID ServiceRoutine;
  95 } KIDT_INIT, *PKIDT_INIT;
  96
  97 #include <pshpack1.h>
  98 typedef struct _KI_INTERRUPT_DISPATCH_ENTRY
  99 {
 100     UCHAR _Op_nop;
 101     UCHAR _Op_push;
 102     UCHAR _Vector;
 103     UCHAR _Op_jmp;
 104     ULONG RelativeAddress;
 105 } KI_INTERRUPT_DISPATCH_ENTRY, *PKI_INTERRUPT_DISPATCH_ENTRY;
 106 #include <poppack.h>
 107
 108 extern ULONG KeI386NpxPresent;
 109 extern ULONG KeI386XMMIPresent;
 110 extern ULONG KeI386FxsrPresent;
 111 extern ULONG KeI386CpuType;
 112 extern ULONG KeI386CpuStep;
 113
 114 //
 115 // INT3 is 1 byte long
 116 //
 117 #define KD_BREAKPOINT_TYPE        UCHAR
 118 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
 119 #define KD_BREAKPOINT_VALUE       0xCC
 120
 121 //
 122 // One-liners for getting and setting special purpose registers in portable code
 123 //
 124 FORCEINLINE
 125 ULONG_PTR
 126 KeGetContextPc(PCONTEXT Context)
 127 {
 128     return Context->Rip;
 129 }
 130
 131 FORCEINLINE
 132 VOID
 133 KeSetContextPc(PCONTEXT Context, ULONG_PTR ProgramCounter)
 134 {
 135     Context->Rip = ProgramCounter;
 136 }
 137
 138 FORCEINLINE
 139 ULONG_PTR
 140 KeGetContextReturnRegister(PCONTEXT Context)
 141 {
 142     return Context->Rax;
 143 }
 144
 145 FORCEINLINE
 146 VOID
 147 KeSetContextReturnRegister(PCONTEXT Context, ULONG_PTR ReturnValue)
 148 {
 149     Context->Rax = ReturnValue;
 150 }
 151
 152 FORCEINLINE
 153 ULONG_PTR
 154 KeGetContextStackRegister(PCONTEXT Context)
 155 {
 156     return Context->Rsp;
 157 }
 158
 159 FORCEINLINE
 160 ULONG_PTR
 161 KeGetContextFrameRegister(PCONTEXT Context)
 162 {
 163     return Context->Rbp;
 164 }
 165
 166 FORCEINLINE
 167 VOID
 168 KeSetContextFrameRegister(PCONTEXT Context, ULONG_PTR Frame)
 169 {
 170     Context->Rbp = Frame;
 171 }
 172
 173 FORCEINLINE
 174 ULONG_PTR
 175 KeGetTrapFramePc(PKTRAP_FRAME TrapFrame)
 176 {
 177     return TrapFrame->Rip;
 178 }
 179
 180 FORCEINLINE
 181 PKTRAP_FRAME
 182 KiGetLinkedTrapFrame(PKTRAP_FRAME TrapFrame)
 183 {
 184     return (PKTRAP_FRAME)TrapFrame->TrapFrame;
 185 }
 186
 187 FORCEINLINE
 188 ULONG_PTR
 189 KeGetTrapFrameStackRegister(PKTRAP_FRAME TrapFrame)
 190 {
 191     return TrapFrame->Rsp;
 192 }
 193
 194 FORCEINLINE
 195 ULONG_PTR
 196 KeGetTrapFrameFrameRegister(PKTRAP_FRAME TrapFrame)
 197 {
 198     return TrapFrame->Rbp;
 199 }
 200
 201 //
 202 // Macro to get trap and exception frame from a thread stack
 203 //
 204 #define KeGetTrapFrame(Thread) \
 205     (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
 206                    sizeof(KTRAP_FRAME))
 207
 208 //
 209 // Macro to get context switches from the PRCB
 210 // All architectures but x86 have it in the PRCB's KeContextSwitches
 211 //
 212 #define KeGetContextSwitches(Prcb)  \
 213     (Prcb->KeContextSwitches)
 214
 215 //
 216 // Macro to get the second level cache size field name which differs between
 217 // CISC and RISC architectures, as the former has unified I/D cache
 218 //
 219 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
 220
 221 #define KeGetExceptionFrame(Thread) \
 222     (PKEXCEPTION_FRAME)((ULONG_PTR)KeGetTrapFrame(Thread) - \
 223                         sizeof(KEXCEPTION_FRAME))
 224
 225 //
 226 // Returns the Interrupt State from a Trap Frame.
 227 // ON = TRUE, OFF = FALSE
 228 //
 229 #define KeGetTrapFrameInterruptState(TrapFrame) \
 230         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
 231
 232 /* Diable interrupts and return whether they were enabled before */
 233 FORCEINLINE
 234 BOOLEAN
 235 KeDisableInterrupts(VOID)
 236 {
 237     ULONG_PTR Flags;
 238
 239     /* Get EFLAGS and check if the interrupt bit is set */
 240     Flags = __readeflags();
 241
 242     /* Disable interrupts */
 243     _disable();
 244     return (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE : FALSE;
 245 }
 246
 247 /* Restore previous interrupt state */
 248 FORCEINLINE
 249 VOID
 250 KeRestoreInterrupts(BOOLEAN WereEnabled)
 251 {
 252     if (WereEnabled) _enable();
 253 }
 254
 255 //
 256 // Invalidates the TLB entry for a specified address
 257 //
 258 FORCEINLINE
 259 VOID
 260 KeInvalidateTlbEntry(IN PVOID Address)
 261 {
 262     /* Invalidate the TLB entry for this address */
 263     __invlpg(Address);
 264 }
 265
 266 FORCEINLINE
 267 VOID
 268 KeFlushProcessTb(VOID)
 269 {
 270     /* Flush the TLB by resetting CR3 */
 271     __writecr3(__readcr3());
 272 }
 273
 274 FORCEINLINE
 275 VOID
 276 KeSweepICache(IN PVOID BaseAddress,
 277               IN SIZE_T FlushSize)
 278 {
 279     //
 280     // Always sweep the whole cache
 281     //
 282     UNREFERENCED_PARAMETER(BaseAddress);
 283     UNREFERENCED_PARAMETER(FlushSize);
 284     __wbinvd();
 285 }
 286
 287 FORCEINLINE
 288 VOID
 289 KiRundownThread(IN PKTHREAD Thread)
 290 {
 291     /* Nothing to do */
 292 }
 293
 294 /* Registers an interrupt handler with an IDT vector */
 295 FORCEINLINE
 296 VOID
 297 KeRegisterInterruptHandler(IN ULONG Vector,
 298                            IN PVOID Handler)
 299 {
 300     UCHAR Entry;
 301     PKIDTENTRY64 Idt;
 302
 303     /* Get the entry from the HAL */
 304     Entry = HalVectorToIDTEntry(Vector);
 305
 306     /* Now set the data */
 307     Idt = &KeGetPcr()->IdtBase[Entry];
 308     Idt->OffsetLow = (ULONG_PTR)Handler & 0xffff;
 309     Idt->OffsetMiddle = ((ULONG_PTR)Handler >> 16) & 0xffff;
 310     Idt->OffsetHigh = (ULONG_PTR)Handler >> 32;
 311     Idt->Selector = KGDT64_R0_CODE;
 312     Idt->IstIndex = 0;
 313     Idt->Type = 0x0e;
 314     Idt->Dpl = 0;
 315     Idt->Present = 1;
 316     Idt->Reserved0 = 0;
 317     Idt->Reserved1 = 0;
 318 }
 319
 320 /* Returns the registered interrupt handler for a given IDT vector */
 321 FORCEINLINE
 322 PVOID
 323 KeQueryInterruptHandler(IN ULONG Vector)
 324 {
 325     UCHAR Entry;
 326     PKIDTENTRY64 Idt;
 327
 328     /* Get the entry from the HAL */
 329     Entry = HalVectorToIDTEntry(Vector);
 330
 331     /* Get the IDT entry */
 332     Idt = &KeGetPcr()->IdtBase[Entry];
 333
 334     /* Return the address */
 335     return (PVOID)((ULONG64)Idt->OffsetHigh << 32 |
 336                    (ULONG64)Idt->OffsetMiddle << 16 |
 337                    (ULONG64)Idt->OffsetLow);
 338 }
 339
 340 FORCEINLINE
 341 VOID
 342 KiSendEOI(VOID)
 343 {
 344     /* Write 0 to the apic EOI register */
 345     *((volatile ULONG*)APIC_EOI_REGISTER) = 0;
 346 }
 347
 348 FORCEINLINE
 349 VOID
 350 KiEndInterrupt(IN KIRQL Irql,
 351                IN PKTRAP_FRAME TrapFrame)
 352 {
 353     /* Make sure this is from the clock handler */
 354     ASSERT(TrapFrame->ErrorCode == 0xc10c4);
 355     //KeLowerIrql(Irql);
 356 }
 357
 358 FORCEINLINE
 359 BOOLEAN
 360 KiUserTrap(IN PKTRAP_FRAME TrapFrame)
 361 {
 362     /* Anything else but Ring 0 is Ring 3 */
 363     return !!(TrapFrame->SegCs & MODE_MASK);
 364 }
 365
 366 #define Ki386PerfEnd()
 367
 368 struct _KPCR;
 369
 370 //VOID KiInitializeTss(IN PKTSS Tss, IN UINT64 Stack);
 371
 372 DECLSPEC_NORETURN VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack);
 373 VOID KiDivideErrorFault(VOID);
 374 VOID KiDebugTrapOrFault(VOID);
 375 VOID KiNmiInterrupt(VOID);
 376 VOID KiBreakpointTrap(VOID);
 377 VOID KiOverflowTrap(VOID);
 378 VOID KiBoundFault(VOID);
 379 VOID KiInvalidOpcodeFault(VOID);
 380 VOID KiNpxNotAvailableFault(VOID);
 381 VOID KiDoubleFaultAbort(VOID);
 382 VOID KiNpxSegmentOverrunAbort(VOID);
 383 VOID KiInvalidTssFault(VOID);
 384 VOID KiSegmentNotPresentFault(VOID);
 385 VOID KiStackFault(VOID);
 386 VOID KiGeneralProtectionFault(VOID);
 387 VOID KiPageFault(VOID);
 388 VOID KiFloatingErrorFault(VOID);
 389 VOID KiAlignmentFault(VOID);
 390 VOID KiMcheckAbort(VOID);
 391 VOID KiXmmException(VOID);
 392 VOID KiApcInterrupt(VOID);
 393 VOID KiRaiseAssertion(VOID);
 394 VOID KiDebugServiceTrap(VOID);
 395 VOID KiDpcInterrupt(VOID);
 396 VOID KiIpiInterrupt(VOID);
 397
 398 VOID KiGdtPrepareForApplicationProcessorInit(ULONG Id);
 399 VOID Ki386InitializeLdt(VOID);
 400 VOID Ki386SetProcessorFeatures(VOID);
 401 VOID KiGetCacheInformation(VOID);
 402 VOID KiSetProcessorType(VOID);
 403 ULONG KiGetFeatureBits(VOID);
 404 VOID KiInitializeCpuFeatures(VOID);
 405
 406 ULONG KeAllocateGdtSelector(ULONG Desc[2]);
 407 VOID KeFreeGdtSelector(ULONG Entry);
 408 VOID NtEarlyInitVdm(VOID);
 409 VOID KeApplicationProcessorInitDispatcher(VOID);
 410 VOID KeCreateApplicationProcessorIdleThread(ULONG Id);
 411
 412 VOID
 413 Ke386InitThreadWithContext(PKTHREAD Thread,
 414                            PKSYSTEM_ROUTINE SystemRoutine,
 415                            PKSTART_ROUTINE StartRoutine,
 416                            PVOID StartContext,
 417                            PCONTEXT Context);
 418 #define KeArchInitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context) \
 419   Ke386InitThreadWithContext(Thread,SystemRoutine,StartRoutine,StartContext,Context)
 420
 421 #ifdef _NTOSKRNL_ /* FIXME: Move flags above to NDK instead of here */
 422 VOID
 423 KiThreadStartup(PKSYSTEM_ROUTINE SystemRoutine,
 424                 PKSTART_ROUTINE StartRoutine,
 425                 PVOID StartContext,
 426                 BOOLEAN UserThread,
 427                 KTRAP_FRAME TrapFrame);
 428 #endif
 429
 430 #endif /* __ASM__ */
 431
 432 // HACK
 433 extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount;
 434
 435 // win64 uses DMA macros, this one is not defined
 436 NTHALAPI
 437 NTSTATUS
 438 NTAPI
 439 HalAllocateAdapterChannel(
 440   IN PADAPTER_OBJECT  AdapterObject,
 441   IN PWAIT_CONTEXT_BLOCK  Wcb,
 442   IN ULONG  NumberOfMapRegisters,
 443   IN PDRIVER_CONTROL  ExecutionRoutine);
 444
 445 FORCEINLINE
 446 PULONG_PTR
 447 KiGetUserModeStackAddress(void)
 448 {
 449     return &PsGetCurrentThread()->Tcb.TrapFrame->Rsp;
 450 }
 451
 452 VOID
 453 KiSetTrapContext(
 454     _Out_ PKTRAP_FRAME TrapFrame,
 455     _In_ PCONTEXT Context,
 456     _In_ KPROCESSOR_MODE RequestorMode);
 457
 458 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_AMD64_KE_H */
 459
 460 /* EOF */