8 // Thread Dispatcher Header DebugActive Mask
10 #define DR_MASK(x) (1 << (x))
11 #define DR_REG_MASK 0x4F
14 // INT3 is 1 byte long
16 #define KD_BREAKPOINT_TYPE UCHAR
17 #define KD_BREAKPOINT_SIZE sizeof(UCHAR)
18 #define KD_BREAKPOINT_VALUE 0xCC
21 // Macros for getting and setting special purpose registers in portable code
23 #define KeGetContextPc(Context) \
26 #define KeSetContextPc(Context, ProgramCounter) \
27 ((Context)->Eip = (ProgramCounter))
29 #define KeGetTrapFramePc(TrapFrame) \
32 #define KiGetLinkedTrapFrame(x) \
33 (PKTRAP_FRAME)((x)->Edx)
35 #define KeGetContextReturnRegister(Context) \
38 #define KeSetContextReturnRegister(Context, ReturnValue) \
39 ((Context)->Eax = (ReturnValue))
42 // Macro to get trap and exception frame from a thread stack
44 #define KeGetTrapFrame(Thread) \
45 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
46 sizeof(KTRAP_FRAME) - \
49 #define KeGetExceptionFrame(Thread) \
53 // Macro to get context switches from the PRCB
54 // All architectures but x86 have it in the PRCB's KeContextSwitches
56 #define KeGetContextSwitches(Prcb) \
57 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
60 // Macro to get the second level cache size field name which differs between
61 // CISC and RISC architectures, as the former has unified I/D cache
63 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
66 // Returns the Interrupt State from a Trap Frame.
67 // ON = TRUE, OFF = FALSE
69 #define KeGetTrapFrameInterruptState(TrapFrame) \
70 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
73 // Flags for exiting a trap
75 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
76 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits)
77 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
79 typedef union _KTRAP_EXIT_SKIP_BITS
83 UCHAR SkipPreviousMode
:1;
85 UCHAR SkipVolatiles
:1;
89 } KTRAP_EXIT_SKIP_BITS
, *PKTRAP_EXIT_SKIP_BITS
;
93 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
95 #define PFX_FLAG_ES 0x00000100
96 #define PFX_FLAG_CS 0x00000200
97 #define PFX_FLAG_SS 0x00000400
98 #define PFX_FLAG_DS 0x00000800
99 #define PFX_FLAG_FS 0x00001000
100 #define PFX_FLAG_GS 0x00002000
101 #define PFX_FLAG_OPER32 0x00004000
102 #define PFX_FLAG_ADDR32 0x00008000
103 #define PFX_FLAG_LOCK 0x00010000
104 #define PFX_FLAG_REPNE 0x00020000
105 #define PFX_FLAG_REP 0x00040000
110 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
111 // We need to keep 2 parameters while the original ASM implementation uses 4:
112 // TrapFrame, PrefixFlags, Eip, InstructionSize;
114 // We pass the trap frame, and prefix flags, in our two parameters.
116 // We then realize that since the smallest prefix flag is 0x100, this gives us
117 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
119 // We further realize that we always have access to EIP from the trap frame, and
120 // that if we want the *current instruction* EIP, we simply have to add the
121 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
122 // at now, so we don't need to use the stack to push this parameter.
124 // We actually only care about the *current instruction* EIP in one location,
125 // so although it may be slightly more expensive to re-calculate the EIP one
126 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
127 // which would be forcing stack usage in all other scenarios.
129 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x));
130 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x))
131 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags)
132 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x)
133 #define KiVdmUnhandledOpcode(x) \
136 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \
139 /* Not yet handled */ \
140 UNIMPLEMENTED_DBGBREAK(); \
144 C_ASSERT(NPX_FRAME_LENGTH
== sizeof(FX_SAVE_AREA
));
149 typedef struct _KV86_FRAME
154 } KV86_FRAME
, *PKV86_FRAME
;
157 // Virtual Stack Frame
159 typedef struct _KV8086_STACK_FRAME
161 KTRAP_FRAME TrapFrame
;
162 FX_SAVE_AREA NpxArea
;
164 } KV8086_STACK_FRAME
, *PKV8086_STACK_FRAME
;
167 // Large Pages Support
169 typedef struct _LARGE_IDENTITY_MAP
171 PHARDWARE_PTE TopLevelDirectory
;
173 ULONG_PTR StartAddress
;
176 } LARGE_IDENTITY_MAP
, *PLARGE_IDENTITY_MAP
;
178 /* Diable interrupts and return whether they were enabled before */
181 KeDisableInterrupts(VOID
)
186 /* Get EFLAGS and check if the interrupt bit is set */
187 Flags
= __readeflags();
188 Return
= (Flags
& EFLAGS_INTERRUPT_MASK
) ? TRUE
: FALSE
;
190 /* Disable interrupts */
195 /* Restore previous interrupt state */
198 KeRestoreInterrupts(BOOLEAN WereEnabled
)
200 if (WereEnabled
) _enable();
204 // Registers an interrupt handler with an IDT vector
208 KeRegisterInterruptHandler(IN ULONG Vector
,
213 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
216 // Get the entry from the HAL
218 Entry
= HalVectorToIDTEntry(Vector
);
219 Address
= PtrToUlong(Handler
);
224 Pcr
->IDT
[Entry
].ExtendedOffset
= (USHORT
)(Address
>> 16);
225 Pcr
->IDT
[Entry
].Offset
= (USHORT
)Address
;
229 // Returns the registered interrupt handler for a given IDT vector
233 KeQueryInterruptHandler(IN ULONG Vector
)
235 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
239 // Get the entry from the HAL
241 Entry
= HalVectorToIDTEntry(Vector
);
244 // Read the entry from the IDT
246 return (PVOID
)(((Pcr
->IDT
[Entry
].ExtendedOffset
<< 16) & 0xFFFF0000) |
247 (Pcr
->IDT
[Entry
].Offset
& 0xFFFF));
251 // Invalidates the TLB entry for a specified address
255 KeInvalidateTlbEntry(IN PVOID Address
)
257 /* Invalidate the TLB entry for this address */
263 KeFlushProcessTb(VOID
)
265 /* Flush the TLB by resetting CR3 */
266 __writecr3(__readcr3());
271 KeGetCurrentThread(VOID
)
273 /* Return the current thread */
274 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
279 KiRundownThread(IN PKTHREAD Thread
)
282 /* Check if this is the NPX Thread */
283 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
286 KeGetCurrentPrcb()->NpxThread
= NULL
;
296 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry
, PVOID BaseAddress
)
298 GdtEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)BaseAddress
& 0xFFFF);
299 GdtEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)BaseAddress
>> 16);
300 GdtEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)BaseAddress
>> 24);
305 KiSetTebBase(PKPCR Pcr
, PVOID TebAddress
)
307 Pcr
->NtTib
.Self
= TebAddress
;
308 Ke386SetGdtEntryBase(&Pcr
->GDT
[KGDT_R3_TEB
/ sizeof(KGDTENTRY
)], TebAddress
);
325 KiGetCacheInformation(VOID
);
335 KiIsNpxErrataPresent(
341 KiSetProcessorType(VOID
);
345 KiGetFeatureBits(VOID
);
349 KiThreadStartup(VOID
);
353 Ke386GetGdtEntryThread(
356 IN PKGDTENTRY Descriptor
362 IN FLOATING_SAVE_AREA
*SaveArea
368 IN PKTRAP_FRAME TrapFrame
373 Ki386SetupAndExitToV86Mode(
385 Ki386EnableGlobalPage(
386 IN
volatile ULONG_PTR Context
391 Ki386EnableTargetLargePage(
392 IN
volatile ULONG_PTR Context
397 Ki386CreateIdentityMap(
398 IN PLARGE_IDENTITY_MAP IdentityMap
,
405 Ki386FreeIdentityMap(
406 IN PLARGE_IDENTITY_MAP IdentityMap
411 Ki386EnableCurrentLargePage(
412 IN ULONG_PTR StartAddress
,
418 KiI386PentiumLockErrataFixup(
436 KiAmdK6InitializeMTRR(
442 KiRestoreFastSyscallReturnState(
460 Ki386EnableXMMIExceptions(
467 IN PKTRAP_FRAME TrapFrame
473 IN PKTRAP_FRAME TrapFrame
,
479 Ki386HandleOpcodeV86(
480 IN PKTRAP_FRAME TrapFrame
487 IN PKTRAP_FRAME TrapFrame
492 Ki386BiosCallReturnAddress(
493 IN PKTRAP_FRAME TrapFrame
499 IN PKTRAP_FRAME TrapFrame
505 KiDispatchExceptionFromTrapFrame(
507 IN ULONG_PTR Address
,
508 IN ULONG ParameterCount
,
509 IN ULONG_PTR Parameter1
,
510 IN ULONG_PTR Parameter2
,
511 IN ULONG_PTR Parameter3
,
512 IN PKTRAP_FRAME TrapFrame
516 // Global x86 only Kernel data
518 extern PVOID Ki386IopmSaveArea
;
519 extern ULONG KeI386EFlagsAndMaskV86
;
520 extern ULONG KeI386EFlagsOrMaskV86
;
521 extern BOOLEAN KeI386VirtualIntExtensions
;
522 extern KIDTENTRY KiIdt
[MAXIMUM_IDTVECTOR
+1];
523 extern KDESCRIPTOR KiIdtDescriptor
;
524 extern BOOLEAN KiI386PentiumLockErrataPresent
;
525 extern ULONG KeI386NpxPresent
;
526 extern ULONG KeI386XMMIPresent
;
527 extern ULONG KeI386FxsrPresent
;
528 extern ULONG KiMXCsrMask
;
529 extern ULONG KeI386CpuType
;
530 extern ULONG KeI386CpuStep
;
531 extern ULONG Ke386CacheAlignment
;
532 extern ULONG KiFastSystemCallDisable
;
533 extern UCHAR KiDebugRegisterTrapOffsets
[9];
534 extern UCHAR KiDebugRegisterContextOffsets
[9];
535 extern DECLSPEC_NORETURN VOID __cdecl
KiTrap02(VOID
);
536 extern VOID __cdecl
KiTrap08(VOID
);
537 extern VOID __cdecl
KiTrap13(VOID
);
538 extern VOID __cdecl
KiFastCallEntry(VOID
);
539 extern VOID NTAPI
ExpInterlockedPopEntrySListFault(VOID
);
540 extern VOID NTAPI
ExpInterlockedPopEntrySListResume(VOID
);
541 extern VOID __cdecl
CopyParams(VOID
);
542 extern VOID __cdecl
ReadBatch(VOID
);
543 extern VOID __cdecl
FrRestore(VOID
);
544 extern CHAR KiSystemCallExitBranch
[];
545 extern CHAR KiSystemCallExit
[];
546 extern CHAR KiSystemCallExit2
[];
554 // Returns a thread's FPU save area
558 KiGetThreadNpxArea(IN PKTHREAD Thread
)
560 return (PFX_SAVE_AREA
)((ULONG_PTR
)Thread
->InitialStack
- sizeof(FX_SAVE_AREA
));
564 // Sanitizes a selector
568 Ke386SanitizeSeg(IN ULONG Cs
,
569 IN KPROCESSOR_MODE Mode
)
572 // Check if we're in kernel-mode, and force CPL 0 if so.
573 // Otherwise, force CPL 3.
575 return ((Mode
== KernelMode
) ?
576 (Cs
& (0xFFFF & ~RPL_MASK
)) :
577 (RPL_MASK
| (Cs
& 0xFFFF)));
585 Ke386SanitizeFlags(IN ULONG Eflags
,
586 IN KPROCESSOR_MODE Mode
)
589 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
590 // Otherwise, also force interrupt mask on.
592 return ((Mode
== KernelMode
) ?
593 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
594 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
598 // Sanitizes a Debug Register
602 Ke386SanitizeDr(IN PVOID DrAddress
,
603 IN KPROCESSOR_MODE Mode
)
606 // Check if we're in kernel-mode, and return the address directly if so.
607 // Otherwise, make sure it's not inside the kernel-mode address space.
608 // If it is, then clear the address.
610 return ((Mode
== KernelMode
) ? DrAddress
:
611 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
615 // Exception with no arguments
620 KiDispatchException0Args(IN NTSTATUS Code
,
621 IN ULONG_PTR Address
,
622 IN PKTRAP_FRAME TrapFrame
)
624 /* Helper for exceptions with no arguments */
625 KiDispatchExceptionFromTrapFrame(Code
, Address
, 0, 0, 0, 0, TrapFrame
);
629 // Exception with one argument
634 KiDispatchException1Args(IN NTSTATUS Code
,
635 IN ULONG_PTR Address
,
637 IN PKTRAP_FRAME TrapFrame
)
639 /* Helper for exceptions with no arguments */
640 KiDispatchExceptionFromTrapFrame(Code
, Address
, 1, P1
, 0, 0, TrapFrame
);
644 // Exception with two arguments
649 KiDispatchException2Args(IN NTSTATUS Code
,
650 IN ULONG_PTR Address
,
653 IN PKTRAP_FRAME TrapFrame
)
655 /* Helper for exceptions with no arguments */
656 KiDispatchExceptionFromTrapFrame(Code
, Address
, 2, P1
, P2
, 0, TrapFrame
);
660 // Performs a system call
664 * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
665 * and then calls the function associated with the system call.
667 * It's done in assembly for two reasons: we need to muck with the stack,
668 * and the call itself restores the stack back for us. The only way to do
669 * this in C is to do manual C handlers for every possible number of args on
670 * the stack, and then have the handler issue a call by pointer. This is
671 * wasteful since it'll basically push the values twice and require another
672 * level of call indirection.
674 * The ARM kernel currently does this, but it should probably be changed
675 * later to function like this as well.
681 KiSystemCallTrampoline(IN PVOID Handler
,
690 "movl %%esp, %%edi\n"
700 : "%esp", "%esi", "%edi"
704 #elif defined(_MSC_VER)
707 KiSystemCallTrampoline(IN PVOID Handler
,
722 /* Return with result in EAX */
725 #error Unknown Compiler
730 // Checks for pending APCs
734 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame
)
739 /* Check for V8086 or user-mode trap */
740 if ((TrapFrame
->EFlags
& EFLAGS_V86_MASK
) || (KiUserTrap(TrapFrame
)))
743 Thread
= KeGetCurrentThread();
746 /* Turn off the alerted state for kernel mode */
747 Thread
->Alerted
[KernelMode
] = FALSE
;
749 /* Are there pending user APCs? */
750 if (!Thread
->ApcState
.UserApcPending
) break;
752 /* Raise to APC level and enable interrupts */
753 OldIrql
= KfRaiseIrql(APC_LEVEL
);
757 KiDeliverApc(UserMode
, NULL
, TrapFrame
);
759 /* Restore IRQL and disable interrupts once again */
760 KfLowerIrql(OldIrql
);
767 // Converts a base thread to a GUI thread
772 KiConvertToGuiThread(VOID
)
778 * Converting to a GUI thread safely updates ESP in-place as well as the
779 * current Thread->TrapFrame and EBP when KeSwitchKernelStack is called.
781 * However, PsConvertToGuiThread "helpfully" restores EBP to the original
782 * caller's value, since it is considered a nonvolatile register. As such,
783 * as soon as we're back after the conversion and we try to store the result
784 * which will probably be in some stack variable (EBP-based), we'll crash as
785 * we are touching the de-allocated non-expanded stack.
787 * Thus we need a way to update our EBP before EBP is touched, and the only
788 * way to guarantee this is to do the call itself in assembly, use the EAX
789 * register to store the result, fixup EBP, and then let the C code continue
797 "call _PsConvertToGuiThread@0\n\t"
800 : "=a"(Result
), "=r"(StackFrame
)
802 : "%esp", "%ecx", "%edx", "memory"
806 #elif defined(_MSC_VER)
809 KiConvertToGuiThread(VOID
);
811 #error Unknown Compiler
815 // Switches from boot loader to initial kernel stack
819 KiSwitchToBootStack(IN ULONG_PTR InitialStack
)
821 /* We have to switch to a new stack before continuing kernel initialization */
828 "jmp _KiSystemStartupBootStack@0\n"
831 "i"(NPX_FRAME_LENGTH
+ KTRAP_FRAME_ALIGN
+ KTRAP_FRAME_LENGTH
),
832 "i"(CR0_EM
| CR0_TS
| CR0_MP
)
835 #elif defined(_MSC_VER)
836 VOID NTAPI
KiSystemStartupBootStack(VOID
);
839 mov esp
, InitialStack
840 sub esp
, (NPX_FRAME_LENGTH
+ KTRAP_FRAME_ALIGN
+ KTRAP_FRAME_LENGTH
)
841 push (CR0_EM
| CR0_TS
| CR0_MP
)
842 jmp KiSystemStartupBootStack
845 #error Unknown Compiler
850 // Emits the iret instruction for C code
857 #if defined(__GNUC__)
862 #elif defined(_MSC_VER)
868 #error Unsupported compiler
874 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
875 // initiates the end by calling back into the HAL and exiting the trap here.
879 KiEndInterrupt(IN KIRQL Irql
,
880 IN PKTRAP_FRAME TrapFrame
)
882 /* Disable interrupts and end the interrupt */
884 HalEndSystemInterrupt(Irql
, TrapFrame
);
886 /* Exit the interrupt */
887 KiEoiHelper(TrapFrame
);
897 extern ULONGLONG BootCyclesEnd
, BootCycles
;
898 BootCyclesEnd
= __rdtsc();
899 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd
- BootCycles
);
900 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
901 KeGetCurrentPrcb()->InterruptCount
,
902 KeGetCurrentPrcb()->KeSystemCalls
,
903 KeGetContextSwitches(KeGetCurrentPrcb()));
908 KiGetUserModeStackAddress(void)
910 return &(KeGetCurrentThread()->TrapFrame
->HardwareEsp
);