2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/include/i386/asmmacro.S
5 * PURPOSE: Assembly Macros for Spinlocks and common Trap Code
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
9 /* INCLUDES ******************************************************************/
13 // Arguments for TRAP_EPILOG
14 #define FromSystemCall 1
15 #define DoRestorePreviousMode 1
16 #define DoRestoreEverything 1
17 #define DoRestoreSegments 1
18 #define DoRestoreVolatiles 1
19 #define DoPushFakeErrorCode 1
21 #define DoFixupAbios 1
22 #define NotFromSystemCall 0
23 #define DoNotRestorePreviousMode 0
24 #define DoNotRestoreEverything 0
25 #define DoNotRestoreSegments 0
26 #define DoNotRestoreVolatiles 0
27 #define DoNotPushFakeErrorCode 0
28 #define DoNotFixupV86 0
29 #define DoNotFixupAbios 0
32 #define INT_32_DPL0 0x8E00
33 #define INT_32_DPL3 0xEE00
36 // This table contains the prefix flags that are used by V86 emulation
38 .equ PREFIX_FLAG_ES, 0x00000100
39 .equ PREFIX_FLAG_CS, 0x00000200
40 .equ PREFIX_FLAG_SS, 0x00000400
41 .equ PREFIX_FLAG_DS, 0x00000800
42 .equ PREFIX_FLAG_FS, 0x00001000
43 .equ PREFIX_FLAG_GS, 0x00002000
44 .equ PREFIX_FLAG_OPER32, 0x00004000
45 .equ PREFIX_FLAG_ADDR32, 0x00008000
46 .equ PREFIX_FLAG_LOCK, 0x00010000
47 .equ PREFIX_FLAG_REPNE, 0x00020000
48 .equ PREFIX_FLAG_REP, 0x00040000
50 .intel_syntax noprefix
53 // These macros are inlined equivalents of KiAcquire/ReleaseSpinlock, that is,
54 // they will not be compiled into non-SMP builds. Usage is as follows:
58 // ACQUIRE_SPINLOCK(reg, .spin)
59 // <thread-safe code here>
60 // RELEASE_SPINLOCK(reg)
65 // <any necessary steps to be able to jump back safely>
66 / SPIN_ON_LOCK(reg, .BeginYourFunction)
71 #define ACQUIRE_SPINLOCK(x, y) \
72 lock bts dword ptr [x], 0; \
74 #define RELEASE_SPINLOCK(x) mov byte ptr [x], 0
75 #define SPIN_ON_LOCK(x, y) \
77 test dword ptr [x], 1; \
83 #define ACQUIRE_SPINLOCK(x, y)
84 #define RELEASE_SPINLOCK(x)
88 // @name UNHANDLED_PATH
90 // This macro prints out that the current code path is not expected yet
96 .macro UNHANDLED_PATH Reason
104 /* Print debug message */
105 push offset _UnhandledMsg
109 /* Loop indefinitely */
117 // @name UNHANDLED_V86_PATH
119 // This macro prints out that the current code path is for unhandled VDM support
125 .macro UNHANDLED_V86_PATH
129 /* Print debug message */
130 push offset _V86UnhandledMsg
134 /* Loop indefinitely */
141 // This macro creates an IDT entry for the given handler
144 // Pointer to the IDT handler
147 // Descriptor Bits to associate
151 .macro idt Handler, Bits
158 // @name GENERATE_IDT_STUB
160 // This macro creates an IDT entry for an unexpected interrupt handler.
166 .macro GENERATE_IDT_STUB Number
167 idt _KiUnexpectedInterrupt&Number, INT_32_DPL0
171 // @name GENERATE_IDT_STUBS
173 // This macro creates unexpected interrupt IDT entries.
180 .macro GENERATE_IDT_STUBS
189 // @name GENERATE_INT_HANDLER
191 // This macro creates an unexpected interrupt handler.
197 .macro GENERATE_INT_HANDLER Number
198 .func KiUnexpectedInterrupt&Number
199 _KiUnexpectedInterrupt&Number:
200 push PRIMARY_VECTOR_BASE + Number
201 jmp _KiEndUnexpectedRange@0
206 // @name GENERATE_INT_HANDLERS
208 // This macro creates the unexpected interrupt handlers.
215 .macro GENERATE_INT_HANDLERS
218 GENERATE_INT_HANDLER %i
224 // @name GENERATE_TRAP_HANDLER
226 // This macro creates a kernel trap handler.
232 .macro GENERATE_TRAP_HANDLER Name, ErrorCode=1, FastV86=0
235 /* Some traps generate an error code, some don't (thanks, Intel!) */
240 /* Make space for trap frame and save registers before we enter C code */
241 sub esp, KTRAP_FRAME_ERROR_CODE
242 mov [esp+KTRAP_FRAME_EAX], eax
243 mov [esp+KTRAP_FRAME_EBX], ebx
244 mov [esp+KTRAP_FRAME_ECX], ecx
245 mov [esp+KTRAP_FRAME_EDX], edx
246 mov [esp+KTRAP_FRAME_ESI], esi
247 mov [esp+KTRAP_FRAME_EDI], edi
248 mov [esp+KTRAP_FRAME_EBP], ebp
252 * The GPF and Invalid Opcode handlers are performance critical when talking
253 * about V8086 traps, because they control the main flow of execution during
254 * that mode (GPFs will be issued for any privileged instruction we need to
255 * emulate, and invalid opcode might be generated for BOPs).
257 * Because of this, we employ a fast entry/exit macro into V8086 mode since
258 * we can make certain assumptions. We detect, and use, such scenarios when
259 * the V8086 flag is enabled in EFLAGS.
261 * However, because we can land in a GPF handler with an invalid DS/ES, as
262 * some V8086 code could trample this during BIOS calls for example, we must
263 * make sure that we are on a valid DS/ES before dereferencing any pointer.
265 * We fixup DS/ES either in KiEnterTrap (for normal entry/exit) or, for V86,
266 * in KiEnterV86Trap. Notice the problem: we need to detect which of these
267 * to use early on but we can't touch the EFLAGS in the frame because DS/ES
270 * Thankfully SS is always guaranteed valid, so stack dereferences are game!
272 * We therefore read the EFLAGS here, in assembly, where we can touch ESP as
273 * we please. We save this in EDX, which will be used as the second argument
274 * for the FASTCALL C trap entry.
276 * When we make the fast V86 check, we use the parameter instead of the trap
277 * frame, leading us to using the correct trap entry function, which fixes
278 * up DS/ES and lets us go on our merry way...
281 mov edx, [esp+KTRAP_FRAME_EFLAGS]
284 /* Normally we just have one parameter, but fast V86 handlers need two */
294 // @name GENERATE_HAL_INT_HANDLER
296 // This macro creates a HAL hardware interrupt handler.
302 .macro GENERATE_HAL_INT_HANDLER Number
303 .func HalpHardwareInterrupt&Number
304 _HalpHardwareInterrupt&Number:
305 int PRIMARY_VECTOR_BASE + Number
311 // @name GENERATE_HAL_INT_HANDLERS
313 // This macro creates the unexpected interrupt handlers.
319 .macro GENERATE_HAL_INT_HANDLERS
322 GENERATE_HAL_INT_HANDLER %i
328 // @name INVALID_V86_OPCODE
330 // This macro creates one or more entries for unhandled V86 Opcodes
331 // in the V86 Opcode Table.
334 // Number of entries to generate.
338 .macro INVALID_V86_OPCODE count
345 // @name GENERATE_PREFIX_HANDLER
347 // This macro creates a prefix opcode handler.
353 .macro GENERATE_PREFIX_HANDLER Name
354 .func Opcode&Name&PrefixV86
355 _Opcode&Name&PrefixV86:
356 or ebx, PREFIX_FLAG_&Name
357 jmp _OpcodeGenericPrefixV86
362 // @name INVALID_V86_OPCODE
364 // This macro prints out visible message and hangs the computer.
368 // @remark Temporary debugging use.
370 .macro UNHANDLED_V86_OPCODE
371 /* Print debug message, breakpoint and freeze */
373 push offset V86DebugMsg
382 // This macro contains out-of-line code for various Trap Frame Fixups, such as:
384 // - DR Fixup: Loads and restores DR registers.
385 // - V86 Fixup: Loads and restores V86 segments.
386 // - ABIOS Fixup: Loads and restores the ABIOS state and stack.
390 // @remark ebp = PKTRAP_FRAME
392 .macro TRAP_FIXUPS Label, EndLabel, V86Fix, AbiosFix
395 /* Check if this was V86 mode */
396 test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
399 /* Check if it was user mode */
400 test byte ptr [ebp+KTRAP_FRAME_CS], MODE_MASK
410 mov [ebp+KTRAP_FRAME_DR0], ebx
411 mov [ebp+KTRAP_FRAME_DR1], ecx
412 mov [ebp+KTRAP_FRAME_DR2], edi
420 mov [ebp+KTRAP_FRAME_DR3], ebx
421 mov [ebp+KTRAP_FRAME_DR6], ecx
422 mov [ebp+KTRAP_FRAME_DR7], edi
429 mov edi, fs:[KPCR_PRCB]
432 mov ebx, [edi+KPRCB_DR0]
433 mov ecx, [edi+KPRCB_DR1]
440 mov ebx, [edi+KPRCB_DR2]
441 mov ecx, [edi+KPRCB_DR3]
448 mov ebx, [edi+KPRCB_DR6]
449 mov ecx, [edi+KPRCB_DR7]
464 /* Get V86 segment registers */
465 mov eax, [ebp+KTRAP_FRAME_V86_FS]
466 mov ebx, [ebp+KTRAP_FRAME_V86_GS]
467 mov ecx, [ebp+KTRAP_FRAME_V86_ES]
468 mov edx, [ebp+KTRAP_FRAME_V86_DS]
470 /* Restore them into Protected Mode trap frame */
471 mov [ebp+KTRAP_FRAME_FS], ax
472 mov [ebp+KTRAP_FRAME_GS], bx
473 mov [ebp+KTRAP_FRAME_ES], cx
474 mov [ebp+KTRAP_FRAME_DS], dx
476 /* Go back to mainline code */
482 // @name SET_TF_DEBUG_HEADER
484 // This macro sets up the debug header in the trap frame.
488 // @remark ebp = PKTRAP_FRAME.
489 // edi/ebx = Have been saved and can be used.
491 .macro SET_TF_DEBUG_HEADER
492 /* Get the Debug Trap Frame EBP/EIP */
493 mov ebx, [ebp+KTRAP_FRAME_EBP]
494 mov edi, [ebp+KTRAP_FRAME_EIP]
496 /* Write the debug data */
497 mov [ebp+KTRAP_FRAME_DEBUGPOINTER], edx
498 mov dword ptr [ebp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
499 mov [ebp+KTRAP_FRAME_DEBUGEBP], ebx
500 mov [ebp+KTRAP_FRAME_DEBUGEIP], edi
504 // @name CHECK_FOR_APC_DELIVER
506 // This macro checks if the trapframe indicates a return to user-mode,
507 // and, if so, checks if user-mode APCs should be delivered.
509 // @param PreserveEax
510 // Determines if EAX should be preserved. Implies that the segment
511 // registers will also be saved.
513 // @remark ebp = PKTRAP_FRAME.
514 // ebx = Saved and will be used.
516 .macro CHECK_FOR_APC_DELIVER PreserveEax
517 /* Check for V86 mode */
518 test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
521 /* Deliver APCs only if we were called from user mode */
522 test byte ptr [ebp+KTRAP_FRAME_CS], 1
525 /* Get the current thread */
527 mov ebx, PCR[KPCR_CURRENT_THREAD]
529 /* Make it non-alerted */
530 mov byte ptr [ebx+KTHREAD_ALERTED], 0
532 /* And only if any are actually pending */
533 cmp byte ptr [ebx+KTHREAD_PENDING_USER_APC], 0
536 /* Save pointer to Trap Frame */
540 /* Save some stuff that raising IRQL will kill */
541 mov [ebx+KTRAP_FRAME_EAX], eax
542 mov dword ptr [ebx+KTRAP_FRAME_FS], KGDT_R3_TEB + RPL_MASK
543 mov dword ptr [ebx+KTRAP_FRAME_DS], KGDT_R3_DATA + RPL_MASK
544 mov dword ptr [ebx+KTRAP_FRAME_ES], KGDT_R3_DATA + RPL_MASK
545 mov dword ptr [ebx+KTRAP_FRAME_GS], 0
548 /* Raise IRQL to APC_LEVEL */
560 call _KiDeliverApc@12
562 /* Return to old IRQL */
566 /* Restore EAX (only in volatile case) */
568 mov eax, [ebx+KTRAP_FRAME_EAX]
578 // This macro creates a standard trap entry prologue.
579 // It should be used for entry into any kernel trap (KiTrapXx), but not for
580 // system calls, which require special handling.
583 // Identifying name of the caller function; will be used to append
584 // to the name V86 and DR helper functions, which must already exist.
586 // @remark Use as follows:
588 // /* Push fake error code */
591 // /* Enter common prologue */
595 // <Your Trap Code Here>
597 .macro TRAP_PROLOG Label EndLabel
598 /* Just to be safe, clear out the HIWORD, since it's reserved */
599 mov word ptr [esp+2], 0
601 /* Save the non-volatiles */
607 /* Save FS and set it to PCR */
613 /* Save exception list and bogus previous mode */
614 push fs:[KPCR_EXCEPTION_LIST]
617 /* Save volatiles and segment registers */
625 /* Set the R3 data segment */
626 mov ax, KGDT_R3_DATA + RPL_MASK
628 /* Skip debug registers and debug stuff */
631 /* Load the segment registers */
637 /* Check if this interrupt happened in 16-bit mode */
639 jb _Ki16BitStackException
644 /* Check if this was from V86 Mode */
645 test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
649 /* Get current thread */
650 mov ecx, fs:[KPCR_CURRENT_THREAD]
654 and dword ptr [ebp+KTRAP_FRAME_DR7], 0
656 /* Check if the thread was being debugged */
657 test byte ptr [ecx+KTHREAD_DEBUG_ACTIVE], 0xFF
660 /* Set the Trap Frame Debug Header */
668 // This macro creates a standard interrupt entry prologue.
669 // It should be used for entry into any interrupt, including software.
672 // Identifying name of the caller function; will be used to append
673 // to the name V86, ABIOS and DR helper functions, which must exist.
675 // @remark For software interrupts, make sure that a fake INT stack is created.
677 .macro INT_PROLOG Label EndLabel FakeErrorCode
680 /* Save fake error code */
684 /* Save the non-volatiles */
690 /* Skip debug registers and other stuff */
697 mov [esp+KTRAP_FRAME_EAX], eax
698 mov [esp+KTRAP_FRAME_ECX], ecx
699 mov [esp+KTRAP_FRAME_EDX], edx
700 mov dword ptr [esp+KTRAP_FRAME_PREVIOUS_MODE], -1
702 /* Check if this was from V86 Mode */
703 test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
706 /* Check if this was kernel mode */
708 cmp word ptr [esp+KTRAP_FRAME_CS], KGDT_R0_CODE
712 mov word ptr [esp+KTRAP_FRAME_FS], fs
713 mov word ptr [esp+KTRAP_FRAME_DS], ds
714 mov word ptr [esp+KTRAP_FRAME_ES], es
715 mov [esp+KTRAP_FRAME_GS], gs
717 /* Load the segment registers (use OVERRIDE (0x66)) */
719 mov eax, KGDT_R3_DATA | RPL_MASK
728 /* Get the previous exception list */
729 mov ebx, fs:[KPCR_EXCEPTION_LIST]
731 /* Set the exception handler chain terminator */
732 mov dword ptr fs:[KPCR_EXCEPTION_LIST], -1
734 /* Save the previous exception list */
735 mov [esp+KTRAP_FRAME_EXCEPTION_LIST], ebx
738 /* Setup the 16-bit stack */
739 lea eax, [esp+KTRAP_FRAME_ERROR_CODE]
740 lea ecx, [esp+KTRAP_FRAME_EIP]
745 /* Check if this is the ABIOS stack */
746 /* cmp esp, 0x10000*/
749 /* Delete error code */
750 and dword ptr [esp+KTRAP_FRAME_ERROR_CODE], 0
752 /* Get the current thread and clear direction flag */
753 mov ecx, PCR[KPCR_CURRENT_THREAD]
757 and dword ptr [ebp+KTRAP_FRAME_DR7], 0
759 /* Check if the thread was being debugged */
760 test byte ptr [ecx+KTHREAD_DEBUG_ACTIVE], 0xFF
767 /* Save DR registers if needed */
770 /* Set the trap frame debug header */
776 // @name SYSCALL_PROLOG
778 // This macro creates a system call entry prologue.
779 // It should be used for entry into any fast-system call (KiGetTickCount,
780 // KiCallbackReturn, KiRaiseAssertion) and the generic system call handler
784 // Unique label identifying the name of the caller function; will be
785 // used to append to the name of the DR helper function, which must
790 .macro SYSCALL_PROLOG Label EndLabel
791 /* Create a trap frame */
799 /* Load PCR Selector into fs */
804 /* Get a pointer to the current thread */
805 mov esi, PCR[KPCR_CURRENT_THREAD]
807 /* Save the previous exception list */
808 push PCR[KPCR_EXCEPTION_LIST]
810 /* Set the exception handler chain terminator */
811 mov dword ptr PCR[KPCR_EXCEPTION_LIST], -1
813 /* Save the old previous mode */
814 push [esi+KTHREAD_PREVIOUS_MODE]
816 /* Skip the other registers */
819 /* Set the new previous mode based on the saved CS selector */
822 mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], bl
824 /* Go on the Kernel stack frame */
827 /* Save the old trap frame pointer where EDX would be saved */
828 mov ebx, [esi+KTHREAD_TRAP_FRAME]
829 mov [ebp+KTRAP_FRAME_EDX], ebx
832 and dword ptr [ebp+KTRAP_FRAME_DR7], 0
834 /* Check if the thread was being debugged */
835 test byte ptr [esi+KTHREAD_DEBUG_ACTIVE], 0xFF
837 /* Set the thread's trap frame and clear direction flag */
838 mov [esi+KTHREAD_TRAP_FRAME], ebp
841 /* Save DR registers if needed */
844 /* Set the trap frame debug header */
848 /* Enable interrupts */
853 // @name FASTCALL_PROLOG
858 // Unique label identifying the name of the caller function; will be
859 // used to append to the name of the DR helper function, which must
864 .macro FASTCALL_PROLOG Label EndLabel
866 /* Set user selector */
867 mov ecx, KGDT_R3_DATA | RPL_MASK
873 /* Set DS/ES to User Selector */
877 /* Set the current stack to Kernel Stack */
878 mov ecx, PCR[KPCR_TSS]
879 mov esp, [ecx+KTSS_ESP0]
881 /* Set up a fake INT Stack. */
882 push KGDT_R3_DATA + RPL_MASK
883 push edx /* Ring 3 SS:ESP */
884 pushf /* Ring 3 EFLAGS */
885 push 2 /* Ring 0 EFLAGS */
886 add edx, 8 /* Skip user parameter list */
887 popf /* Set our EFLAGS */
888 or dword ptr [esp], EFLAGS_INTERRUPT_MASK /* Re-enable IRQs in EFLAGS, to fake INT */
889 push KGDT_R3_CODE + RPL_MASK
890 push dword ptr ds:KUSER_SHARED_SYSCALL_RET
892 /* Setup the Trap Frame stack */
898 push KGDT_R3_TEB + RPL_MASK
900 /* Save pointer to our PCR */
901 mov ebx, PCR[KPCR_SELF]
903 /* Get a pointer to the current thread */
904 mov esi, [ebx+KPCR_CURRENT_THREAD]
906 /* Set the exception handler chain terminator */
907 push [ebx+KPCR_EXCEPTION_LIST]
908 mov dword ptr [ebx+KPCR_EXCEPTION_LIST], -1
910 /* Use the thread's stack */
911 mov ebp, [esi+KTHREAD_INITIAL_STACK]
913 /* Push previous mode */
916 /* Skip the other registers */
919 /* Make space for us on the stack */
922 /* Write the previous mode */
923 mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], UserMode
930 and dword ptr [ebp+KTRAP_FRAME_DR7], 0
932 /* Check if the thread was being debugged */
933 test byte ptr [esi+KTHREAD_DEBUG_ACTIVE], 0xFF
935 /* Set the thread's trap frame */
936 mov [esi+KTHREAD_TRAP_FRAME], ebp
938 /* Save DR registers if needed */
941 /* Set the trap frame debug header */
945 /* Enable interrupts */
950 // @name V86_TRAP_PROLOG
952 // This macro creates a V86 Trap entry prologue.
953 // It should be used for entry into any fast-system call (KiGetTickCount,
954 // KiCallbackReturn, KiRaiseAssertion) and the generic system call handler
958 // Unique label identifying the name of the caller function; will be
959 // used to append to the name of the DR helper function, which must
964 .macro V86_TRAP_PROLOG Label EndLabel
966 /* Skip everything to the error code */
967 sub esp, KTRAP_FRAME_ERROR_CODE
969 /* Clear the error code */
970 mov word ptr [esp+KTRAP_FRAME_ERROR_CODE+2], 0
972 /* Save the registers we'll trample */
973 mov [esp+KTRAP_FRAME_EBX], ebx
974 mov [esp+KTRAP_FRAME_EAX], eax
975 mov [esp+KTRAP_FRAME_EBP], ebp
976 mov [esp+KTRAP_FRAME_ESI], esi
977 mov [esp+KTRAP_FRAME_EDI], edi
979 /* Save PCR and Ring 3 segments */
981 mov eax, KGDT_R3_DATA + RPL_MASK
983 /* Save ECX and EDX too */
984 mov [esp+KTRAP_FRAME_ECX], ecx
985 mov [esp+KTRAP_FRAME_EDX], edx
987 /* Set debugging markers */
988 mov dword ptr [esp+KTRAP_FRAME_PREVIOUS_MODE], -1
989 mov dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
991 /* Now set segments (use OVERRIDE, 0x66) */
999 /* Set the trap frame in the stack and clear the direction flag */
1003 /* Save the exception list */
1004 mov eax, fs:[KPCR_EXCEPTION_LIST]
1005 mov [esp+KTRAP_FRAME_EXCEPTION_LIST], eax
1007 /* Check if we need debugging */
1009 test eax, ~DR7_RESERVED_MASK
1010 mov [esp+KTRAP_FRAME_DR7], eax
1017 // @name V86_TRAP_EPILOG
1019 // This macro creates an epilogue for leaving V86 traps
1025 .macro V86_TRAP_EPILOG
1027 /* Get the current thread and make it unalerted */
1029 mov ebx, PCR[KPCR_CURRENT_THREAD]
1030 mov byte ptr [ebx+KTHREAD_ALERTED], 0
1032 /* Check if it has User-mode APCs pending */
1033 cmp byte ptr [ebx+KTHREAD_PENDING_USER_APC], 0
1036 /* It doesn't, pop the frame */
1037 add esp, KTRAP_FRAME_EDX
1042 /* Check if DR registers should be restored */
1043 test dword ptr [ebp+KTRAP_FRAME_DR7], ~DR7_RESERVED_MASK
1046 /* Finish popping the rest of the frame, and return to P-mode */
1060 mov esi, [ebp+KTRAP_FRAME_DR0]
1061 mov edi, [ebp+KTRAP_FRAME_DR1]
1066 /* Get DR2 and load DR0-2 */
1067 mov ebx, [ebp+KTRAP_FRAME_DR2]
1073 mov esi, [ebp+KTRAP_FRAME_DR0]
1074 mov edi, [ebp+KTRAP_FRAME_DR1]
1075 mov ebx, [ebp+KTRAP_FRAME_DR7]
1081 jmp V86DebugContinue
1085 /* Raise to APC level */
1089 /* Save KIRQL and deliver APCs */
1095 call _KiDeliverApc@12
1102 /* Check if we're not in V86 anymore */
1103 test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
1108 // @name TRAP_EPILOG
1110 // This macro creates an epilogue for leaving any system trap.
1111 // It is used for exiting system calls, exceptions, interrupts and generic
1114 // @param SystemCall
1115 // Specifies whether this trap will exit a system call. If so, special
1116 // code will be assembled to potentially use SYSEXIT instead of IRETD.
1118 // @param RestorePreviousMode
1119 // Specifies if the previous mode should be restored.
1121 // @param RestoreSegments
1122 // Specifies if the segment registers should be restored.
1124 // @param RestoreVolatiles
1125 // Specifies if the volatile registers should be restored.
1127 // @param RestoreAllRegs
1128 // Specifies if volatiles and segments should both be restored.
1132 .macro TRAP_EPILOG SystemCall, RestorePreviousMode, RestoreSegments, RestoreVolatiles, RestoreAllRegs
1134 /* Assert the flags */
1137 test edx, EFLAGS_INTERRUPT_MASK
1140 /* Assert the stack */
1144 /* Assert the trap frame */
1148 sub dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
1156 /* Assert exception list */
1157 cmp dword ptr PCR[KPCR_EXCEPTION_LIST], 0
1166 /* Get exception list */
1167 mov edx, [esp+KTRAP_FRAME_EXCEPTION_LIST]
1170 /* Assert the saved exception list */
1178 mov PCR[KPCR_EXCEPTION_LIST], edx
1180 .if \RestorePreviousMode
1181 /* Get previous mode */
1182 mov ecx, [esp+KTRAP_FRAME_PREVIOUS_MODE]
1185 /* Assert the saved previous mode */
1192 /* Restore the previous mode */
1193 mov esi, PCR[KPCR_CURRENT_THREAD]
1194 mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], cl
1198 /* Assert the saved previous mode */
1199 mov ecx, [esp+KTRAP_FRAME_PREVIOUS_MODE]
1207 /* Check for debug registers */
1208 test dword ptr [esp+KTRAP_FRAME_DR7], ~DR7_RESERVED_MASK
1213 test dword ptr [esp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
1216 /* Check if the frame was edited */
1217 test word ptr [esp+KTRAP_FRAME_CS], FRAME_EDITED
1221 /* Check the old mode */
1222 cmp word ptr [esp+KTRAP_FRAME_CS], KGDT_R3_CODE + RPL_MASK
1223 bt word ptr [esp+KTRAP_FRAME_CS], 0
1228 .if \RestoreVolatiles
1229 /* Restore volatiles */
1230 mov edx, [esp+KTRAP_FRAME_EDX]
1231 mov ecx, [esp+KTRAP_FRAME_ECX]
1232 mov eax, [esp+KTRAP_FRAME_EAX]
1235 /* Check if we were called from kernel mode */
1236 cmp word ptr [ebp+KTRAP_FRAME_CS], KGDT_R0_CODE
1239 .if \RestoreSegments
1240 /* Restore segment registers */
1241 lea esp, [ebp+KTRAP_FRAME_GS]
1249 lea esp, [ebp+KTRAP_FRAME_FS]
1253 /* Skip debug information and unsaved registers */
1254 lea esp, [ebp+KTRAP_FRAME_EDI]
1260 /* Check for ABIOS */
1261 cmp word ptr [esp+8], 0x80
1264 /* Pop error code */
1268 /* Check if previous CS is from user-mode */
1269 test dword ptr [esp+4], 1
1271 /* It is, so use Fast Exit */
1274 /* Jump back to stub */
1286 /* Is SYSEXIT Supported/Wanted? */
1287 cmp dword ptr ss:[_KiFastSystemCallDisable], 0
1289 test dword ptr [esp+8], EFLAGS_TF
1292 /* Restore FS to TIB */
1293 mov ecx, KGDT_R3_TEB + RPL_MASK
1296 /* We will be cleaning up the stack ourselves */
1297 pop edx /* New Ring 3 EIP */
1298 add esp, 4 /* Skip Ring 3 DS */
1299 and dword ptr [esp], 0xfffffdff /* Remove EFLAGS_INTERRUPT_MASK from EFLAGS */
1300 popf /* Restore old EFLAGS */
1301 pop ecx /* Old Ring 3 SS:ESP */
1305 * ECX points to the old User Stack.
1306 * EDX points to the instruction to execute in usermode after the sysenter
1315 mov eax, [esp+KTRAP_FRAME_EAX]
1317 /* Skip registers */
1320 /* Restore segments and volatiles */
1327 /* Jump back to mainline code */
1333 /* Fix up the mask */
1334 add dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
1342 /* Check if this was V86 mode */
1343 test dword ptr [esp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
1346 /* Check if it was user mode */
1347 test word ptr [esp+KTRAP_FRAME_CS], MODE_MASK
1356 mov esi, [ebp+KTRAP_FRAME_DR0]
1357 mov edi, [ebp+KTRAP_FRAME_DR1]
1358 mov ebx, [ebp+KTRAP_FRAME_DR2]
1366 mov esi, [ebp+KTRAP_FRAME_DR3]
1367 mov edi, [ebp+KTRAP_FRAME_DR6]
1368 mov ebx, [ebp+KTRAP_FRAME_DR7]
1377 /* Restore real CS value */
1378 mov ebx, [esp+KTRAP_FRAME_TEMPCS]
1379 mov [esp+KTRAP_FRAME_CS], ebx
1382 * If ESP was modified, then a special interrupt exit stack
1383 * must be created to "update" ESP's value in a legal manner
1385 mov ebx, [esp+KTRAP_FRAME_TEMPESP]
1387 mov [esp+KTRAP_FRAME_ERROR_CODE], ebx
1389 /* Copy Interrupt Stack */
1390 mov esi, [esp+KTRAP_FRAME_EFLAGS]
1392 mov esi, [esp+KTRAP_FRAME_CS]
1394 mov esi, [esp+KTRAP_FRAME_EIP]
1397 .if \RestoreVolatiles
1398 /* Restore volatiles */
1399 mov eax, [esp+KTRAP_FRAME_EAX]
1400 mov edx, [esp+KTRAP_FRAME_EDX]
1401 mov ecx, [esp+KTRAP_FRAME_ECX]
1405 add esp, KTRAP_FRAME_EDI
1417 // This macro creates an epilogue for leaving any system trap.
1418 // It is used for exiting system calls, exceptions, interrupts and generic
1421 // @param Spurious - TRUE if the interrupt was unexpected and spurious.
1425 .macro INT_EPILOG Spurious
1428 /* Just exit the trap */
1429 jmp _Kei386EoiHelper@0
1431 /* Disable interrupts */
1434 /* End the interrupt and do EOI */
1435 call _HalEndSystemInterrupt@8
1436 jmp _Kei386EoiHelper@0
1442 .macro VERIFY_INT Label
1443 /* Get the current time and mask it to 192 ticks */
1444 mov eax, _KeTickCount
1447 /* Check if we're in the same tick area */
1448 cmp eax, dword ptr [edi+KINTERRUPT_TICK_COUNT]
1452 /* If we got here, then our count is too large */
1453 dec word ptr [edi+KINTERRUPT_DISPATCH_COUNT]
1458 .macro VERIFY_INT_END Label, Info
1461 /* Decrement the dispatch count and check if we should bug check */
1462 dec word ptr [edi+KINTERRUPT_DISPATCH_COUNT+2]
1465 /* Update the tick count */
1467 mov [edi+KINTERRUPT_TICK_COUNT], eax
1471 /* Check if the debugger is enabled */
1472 cmp byte ptr __KdDebuggerEnabled, 0
1475 /* It isn't, bugcheck */
1478 push [edi+KINTERRUPT_SERVICE_CONTEXT]
1479 push [edi+KINTERRUPT_SERVICE_ROUTINE]
1480 push HARDWARE_INTERRUPT_STORM
1481 call _KeBugCheckEx@20
1484 /* Debugger enabled, do a debug print + break instead */
1485 push [edi+KINTERRUPT_SERVICE_ROUTINE]
1486 push offset _IsrOverflowMsg
1491 /* Breakpoint handled, get the new tick count */
1492 mov eax, _KeTickCount
1496 /* Reset tick count */
1497 mov dword ptr [edi+KINTERRUPT_TICK_COUNT], eax
1498 mov word ptr [edi+KINTERRUPT_DISPATCH_COUNT+2], 64
1501 /* Put default overflow count and continue */
1502 mov ax, _KiISROverflow
1503 mov word ptr [edi+KINTERRUPT_DISPATCH_COUNT], ax
1507 /* Check if we wrapped */
1509 cmp eax, [edi+KINTERRUPT_TICK_COUNT]
1512 /* We did, start over */
1513 mov eax, _KeTickCount
1519 /* We don't verify interrupts on retail builds */
1520 .macro VERIFY_INT Label
1522 .macro VERIFY_INT_END Label, Info