8 // Thread Dispatcher Header DebugActive Mask
10 #define DR_MASK(x) (1 << (x))
11 #define DR_REG_MASK 0x4F
13 #define IMAGE_FILE_MACHINE_ARCHITECTURE IMAGE_FILE_MACHINE_I386
16 // INT3 is 1 byte long
18 #define KD_BREAKPOINT_TYPE UCHAR
19 #define KD_BREAKPOINT_SIZE sizeof(UCHAR)
20 #define KD_BREAKPOINT_VALUE 0xCC
23 // Macros for getting and setting special purpose registers in portable code
25 #define KeGetContextPc(Context) \
28 #define KeSetContextPc(Context, ProgramCounter) \
29 ((Context)->Eip = (ProgramCounter))
31 #define KeGetTrapFramePc(TrapFrame) \
34 #define KiGetLinkedTrapFrame(x) \
35 (PKTRAP_FRAME)((x)->Edx)
37 #define KeGetContextReturnRegister(Context) \
40 #define KeSetContextReturnRegister(Context, ReturnValue) \
41 ((Context)->Eax = (ReturnValue))
44 // Macro to get trap and exception frame from a thread stack
46 #define KeGetTrapFrame(Thread) \
47 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
48 sizeof(KTRAP_FRAME) - \
51 #define KeGetExceptionFrame(Thread) \
55 // Macro to get context switches from the PRCB
56 // All architectures but x86 have it in the PRCB's KeContextSwitches
58 #define KeGetContextSwitches(Prcb) \
59 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
62 // Macro to get the second level cache size field name which differs between
63 // CISC and RISC architectures, as the former has unified I/D cache
65 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
68 // Returns the Interrupt State from a Trap Frame.
69 // ON = TRUE, OFF = FALSE
71 #define KeGetTrapFrameInterruptState(TrapFrame) \
72 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
75 // Flags for exiting a trap
77 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
78 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits)
79 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
81 typedef union _KTRAP_EXIT_SKIP_BITS
85 UCHAR SkipPreviousMode
:1;
87 UCHAR SkipVolatiles
:1;
91 } KTRAP_EXIT_SKIP_BITS
, *PKTRAP_EXIT_SKIP_BITS
;
95 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
97 #define PFX_FLAG_ES 0x00000100
98 #define PFX_FLAG_CS 0x00000200
99 #define PFX_FLAG_SS 0x00000400
100 #define PFX_FLAG_DS 0x00000800
101 #define PFX_FLAG_FS 0x00001000
102 #define PFX_FLAG_GS 0x00002000
103 #define PFX_FLAG_OPER32 0x00004000
104 #define PFX_FLAG_ADDR32 0x00008000
105 #define PFX_FLAG_LOCK 0x00010000
106 #define PFX_FLAG_REPNE 0x00020000
107 #define PFX_FLAG_REP 0x00040000
112 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
113 // We need to keep 2 parameters while the original ASM implementation uses 4:
114 // TrapFrame, PrefixFlags, Eip, InstructionSize;
116 // We pass the trap frame, and prefix flags, in our two parameters.
118 // We then realize that since the smallest prefix flag is 0x100, this gives us
119 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
121 // We further realize that we always have access to EIP from the trap frame, and
122 // that if we want the *current instruction* EIP, we simply have to add the
123 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
124 // at now, so we don't need to use the stack to push this parameter.
126 // We actually only care about the *current instruction* EIP in one location,
127 // so although it may be slightly more expensive to re-calculate the EIP one
128 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
129 // which would be forcing stack usage in all other scenarios.
131 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x));
132 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x))
133 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags)
134 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x)
135 #define KiVdmUnhandledOpcode(x) \
138 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \
141 /* Not yet handled */ \
147 C_ASSERT(NPX_FRAME_LENGTH
== sizeof(FX_SAVE_AREA
));
152 typedef struct _KV86_FRAME
157 } KV86_FRAME
, *PKV86_FRAME
;
160 // Virtual Stack Frame
162 typedef struct _KV8086_STACK_FRAME
164 KTRAP_FRAME TrapFrame
;
165 FX_SAVE_AREA NpxArea
;
167 } KV8086_STACK_FRAME
, *PKV8086_STACK_FRAME
;
170 // Registers an interrupt handler with an IDT vector
174 KeRegisterInterruptHandler(IN ULONG Vector
,
179 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
182 // Get the entry from the HAL
184 Entry
= HalVectorToIDTEntry(Vector
);
185 Address
= PtrToUlong(Handler
);
190 Pcr
->IDT
[Entry
].ExtendedOffset
= (USHORT
)(Address
>> 16);
191 Pcr
->IDT
[Entry
].Offset
= (USHORT
)Address
;
195 // Returns the registered interrupt handler for a given IDT vector
199 KeQueryInterruptHandler(IN ULONG Vector
)
201 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
205 // Get the entry from the HAL
207 Entry
= HalVectorToIDTEntry(Vector
);
210 // Read the entry from the IDT
212 return (PVOID
)(((Pcr
->IDT
[Entry
].ExtendedOffset
<< 16) & 0xFFFF0000) |
213 (Pcr
->IDT
[Entry
].Offset
& 0xFFFF));
217 // Invalidates the TLB entry for a specified address
221 KeInvalidateTlbEntry(IN PVOID Address
)
223 /* Invalidate the TLB entry for this address */
229 KeFlushProcessTb(VOID
)
231 /* Flush the TLB by resetting CR3 */
232 __writecr3(__readcr3());
237 KeGetCurrentThread(VOID
)
239 /* Return the current thread */
240 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
245 KiRundownThread(IN PKTHREAD Thread
)
248 /* Check if this is the NPX Thread */
249 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
252 KeGetCurrentPrcb()->NpxThread
= NULL
;
274 KiGetCacheInformation(VOID
);
284 KiIsNpxErrataPresent(
290 KiSetProcessorType(VOID
);
294 KiGetFeatureBits(VOID
);
298 KiThreadStartup(VOID
);
302 Ke386GetGdtEntryThread(
305 IN PKGDTENTRY Descriptor
311 IN FLOATING_SAVE_AREA
*SaveArea
317 IN PKTRAP_FRAME TrapFrame
322 Ki386SetupAndExitToV86Mode(
334 Ki386EnableGlobalPage(
335 IN
volatile ULONG_PTR Context
340 KiI386PentiumLockErrataFixup(
358 KiAmdK6InitializeMTRR(
364 KiRestoreFastSyscallReturnState(
382 Ki386EnableXMMIExceptions(
389 IN PKTRAP_FRAME TrapFrame
395 IN PKTRAP_FRAME TrapFrame
,
401 Ki386HandleOpcodeV86(
402 IN PKTRAP_FRAME TrapFrame
409 IN PKTRAP_FRAME TrapFrame
414 Ki386BiosCallReturnAddress(
415 IN PKTRAP_FRAME TrapFrame
421 IN PKTRAP_FRAME TrapFrame
427 KiDispatchExceptionFromTrapFrame(
429 IN ULONG_PTR Address
,
430 IN ULONG ParameterCount
,
431 IN ULONG_PTR Parameter1
,
432 IN ULONG_PTR Parameter2
,
433 IN ULONG_PTR Parameter3
,
434 IN PKTRAP_FRAME TrapFrame
438 // Global x86 only Kernel data
440 extern PVOID Ki386IopmSaveArea
;
441 extern ULONG KeI386EFlagsAndMaskV86
;
442 extern ULONG KeI386EFlagsOrMaskV86
;
443 extern BOOLEAN KeI386VirtualIntExtensions
;
444 extern KIDTENTRY KiIdt
[MAXIMUM_IDTVECTOR
];
445 extern KDESCRIPTOR KiIdtDescriptor
;
446 extern BOOLEAN KiI386PentiumLockErrataPresent
;
447 extern ULONG KeI386NpxPresent
;
448 extern ULONG KeI386XMMIPresent
;
449 extern ULONG KeI386FxsrPresent
;
450 extern ULONG KiMXCsrMask
;
451 extern ULONG KeI386CpuType
;
452 extern ULONG KeI386CpuStep
;
453 extern ULONG Ke386CacheAlignment
;
454 extern ULONG KiFastSystemCallDisable
;
455 extern UCHAR KiDebugRegisterTrapOffsets
[9];
456 extern UCHAR KiDebugRegisterContextOffsets
[9];
457 extern DECLSPEC_NORETURN VOID __cdecl
KiTrap02(VOID
);
458 extern VOID __cdecl
KiTrap08(VOID
);
459 extern VOID __cdecl
KiTrap13(VOID
);
460 extern VOID __cdecl
KiFastCallEntry(VOID
);
461 extern VOID NTAPI
ExpInterlockedPopEntrySListFault(VOID
);
462 extern VOID __cdecl
CopyParams(VOID
);
463 extern VOID __cdecl
ReadBatch(VOID
);
464 extern VOID __cdecl
FrRestore(VOID
);
465 extern CHAR KiSystemCallExitBranch
[];
466 extern CHAR KiSystemCallExit
[];
467 extern CHAR KiSystemCallExit2
[];
472 #include "../trap_x.h"
475 // Returns a thread's FPU save area
479 KiGetThreadNpxArea(IN PKTHREAD Thread
)
481 return (PFX_SAVE_AREA
)((ULONG_PTR
)Thread
->InitialStack
- sizeof(FX_SAVE_AREA
));
485 // Sanitizes a selector
489 Ke386SanitizeSeg(IN ULONG Cs
,
490 IN KPROCESSOR_MODE Mode
)
493 // Check if we're in kernel-mode, and force CPL 0 if so.
494 // Otherwise, force CPL 3.
496 return ((Mode
== KernelMode
) ?
497 (Cs
& (0xFFFF & ~RPL_MASK
)) :
498 (RPL_MASK
| (Cs
& 0xFFFF)));
506 Ke386SanitizeFlags(IN ULONG Eflags
,
507 IN KPROCESSOR_MODE Mode
)
510 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
511 // Otherwise, also force interrupt mask on.
513 return ((Mode
== KernelMode
) ?
514 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
515 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
519 // Gets a DR register from a CONTEXT structure
523 KiDrFromContext(IN ULONG Dr
,
526 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
530 // Gets a DR register from a KTRAP_FRAME structure
534 KiDrFromTrapFrame(IN ULONG Dr
,
535 IN PKTRAP_FRAME TrapFrame
)
537 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
541 // Sanitizes a Debug Register
545 Ke386SanitizeDr(IN PVOID DrAddress
,
546 IN KPROCESSOR_MODE Mode
)
549 // Check if we're in kernel-mode, and return the address directly if so.
550 // Otherwise, make sure it's not inside the kernel-mode address space.
551 // If it is, then clear the address.
553 return ((Mode
== KernelMode
) ? DrAddress
:
554 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
558 // Exception with no arguments
563 KiDispatchException0Args(IN NTSTATUS Code
,
564 IN ULONG_PTR Address
,
565 IN PKTRAP_FRAME TrapFrame
)
567 /* Helper for exceptions with no arguments */
568 KiDispatchExceptionFromTrapFrame(Code
, Address
, 0, 0, 0, 0, TrapFrame
);
572 // Exception with one argument
577 KiDispatchException1Args(IN NTSTATUS Code
,
578 IN ULONG_PTR Address
,
580 IN PKTRAP_FRAME TrapFrame
)
582 /* Helper for exceptions with no arguments */
583 KiDispatchExceptionFromTrapFrame(Code
, Address
, 1, P1
, 0, 0, TrapFrame
);
587 // Exception with two arguments
592 KiDispatchException2Args(IN NTSTATUS Code
,
593 IN ULONG_PTR Address
,
596 IN PKTRAP_FRAME TrapFrame
)
598 /* Helper for exceptions with no arguments */
599 KiDispatchExceptionFromTrapFrame(Code
, Address
, 2, P1
, P2
, 0, TrapFrame
);
603 // Performs a system call
607 KiSystemCallTrampoline(IN PVOID Handler
,
614 * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
615 * and then calls the function associated with the system call.
617 * It's done in assembly for two reasons: we need to muck with the stack,
618 * and the call itself restores the stack back for us. The only way to do
619 * this in C is to do manual C handlers for every possible number of args on
620 * the stack, and then have the handler issue a call by pointer. This is
621 * wasteful since it'll basically push the values twice and require another
622 * level of call indirection.
624 * The ARM kernel currently does this, but it should probably be changed
625 * later to function like this as well.
632 "movl %%esp, %%edi\n"
642 : "%esp", "%esi", "%edi"
644 #elif defined(_MSC_VER)
658 #error Unknown Compiler
665 // Checks for pending APCs
669 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame
)
674 /* Check for V8086 or user-mode trap */
675 if ((TrapFrame
->EFlags
& EFLAGS_V86_MASK
) || (KiUserTrap(TrapFrame
)))
678 Thread
= KeGetCurrentThread();
681 /* Turn off the alerted state for kernel mode */
682 Thread
->Alerted
[KernelMode
] = FALSE
;
684 /* Are there pending user APCs? */
685 if (!Thread
->ApcState
.UserApcPending
) break;
687 /* Raise to APC level and enable interrupts */
688 OldIrql
= KfRaiseIrql(APC_LEVEL
);
692 KiDeliverApc(UserMode
, NULL
, TrapFrame
);
694 /* Restore IRQL and disable interrupts once again */
695 KfLowerIrql(OldIrql
);
702 // Converts a base thread to a GUI thread
706 KiConvertToGuiThread(VOID
)
712 * Converting to a GUI thread safely updates ESP in-place as well as the
713 * current Thread->TrapFrame and EBP when KeSwitchKernelStack is called.
715 * However, PsConvertToGuiThread "helpfully" restores EBP to the original
716 * caller's value, since it is considered a nonvolatile register. As such,
717 * as soon as we're back after the conversion and we try to store the result
718 * which will probably be in some stack variable (EBP-based), we'll crash as
719 * we are touching the de-allocated non-expanded stack.
721 * Thus we need a way to update our EBP before EBP is touched, and the only
722 * way to guarantee this is to do the call itself in assembly, use the EAX
723 * register to store the result, fixup EBP, and then let the C code continue
732 "call _PsConvertToGuiThread@0\n"
736 : "=r"(Result
), "=r"(StackFrame
)
738 : "%esp", "%ecx", "%edx", "memory"
740 #elif defined(_MSC_VER)
741 NTSTATUS NTAPI
PsConvertToGuiThread(VOID
);
746 call PsConvertToGuiThread
752 #error Unknown Compiler
758 // Switches from boot loader to initial kernel stack
762 KiSwitchToBootStack(IN ULONG_PTR InitialStack
)
764 /* We have to switch to a new stack before continuing kernel initialization */
771 "jmp _KiSystemStartupBootStack@0\n"
774 "i"(NPX_FRAME_LENGTH
+ KTRAP_FRAME_ALIGN
+ KTRAP_FRAME_LENGTH
),
775 "i"(CR0_EM
| CR0_TS
| CR0_MP
)
778 #elif defined(_MSC_VER)
779 VOID NTAPI
KiSystemStartupBootStack(VOID
);
782 mov esp
, InitialStack
783 sub esp
, (NPX_FRAME_LENGTH
+ KTRAP_FRAME_ALIGN
+ KTRAP_FRAME_LENGTH
)
784 push (CR0_EM
| CR0_TS
| CR0_MP
)
785 jmp KiSystemStartupBootStack
788 #error Unknown Compiler
793 // Emits the iret instruction for C code
800 #if defined(__GNUC__)
805 #elif defined(_MSC_VER)
811 #error Unsupported compiler
817 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
818 // initiates the end by calling back into the HAL and exiting the trap here.
822 KiEndInterrupt(IN KIRQL Irql
,
823 IN PKTRAP_FRAME TrapFrame
)
825 /* Disable interrupts and end the interrupt */
827 HalEndSystemInterrupt(Irql
, TrapFrame
);
829 /* Exit the interrupt */
830 KiEoiHelper(TrapFrame
);
840 extern ULONGLONG BootCyclesEnd
, BootCycles
;
841 BootCyclesEnd
= __rdtsc();
842 DbgPrint("Boot took %I64d cycles!\n", BootCyclesEnd
- BootCycles
);
843 DbgPrint("Interrupts: %d System Calls: %d Context Switches: %d\n",
844 KeGetCurrentPrcb()->InterruptCount
,
845 KeGetCurrentPrcb()->KeSystemCalls
,
846 KeGetContextSwitches(KeGetCurrentPrcb()));