2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/include/trap_x.h
5 * PURPOSE: Internal Inlined Functions for the Trap Handling Code
6 * PROGRAMMERS: ReactOS Portable Systems Group
12 // Unreachable code hint for GCC 4.5.x, older GCC versions, and MSVC
15 #if __GNUC__ * 100 + __GNUC_MINOR__ >= 405
16 #define UNREACHABLE __builtin_unreachable()
18 #define UNREACHABLE __builtin_trap()
21 #define UNREACHABLE __assume(0)
31 KiDumpTrapFrame(IN PKTRAP_FRAME TrapFrame
)
33 /* Dump the whole thing */
34 DbgPrint("DbgEbp: %x\n", TrapFrame
->DbgEbp
);
35 DbgPrint("DbgEip: %x\n", TrapFrame
->DbgEip
);
36 DbgPrint("DbgArgMark: %x\n", TrapFrame
->DbgArgMark
);
37 DbgPrint("DbgArgPointer: %x\n", TrapFrame
->DbgArgPointer
);
38 DbgPrint("TempSegCs: %x\n", TrapFrame
->TempSegCs
);
39 DbgPrint("TempEsp: %x\n", TrapFrame
->TempEsp
);
40 DbgPrint("Dr0: %x\n", TrapFrame
->Dr0
);
41 DbgPrint("Dr1: %x\n", TrapFrame
->Dr1
);
42 DbgPrint("Dr2: %x\n", TrapFrame
->Dr2
);
43 DbgPrint("Dr3: %x\n", TrapFrame
->Dr3
);
44 DbgPrint("Dr6: %x\n", TrapFrame
->Dr6
);
45 DbgPrint("Dr7: %x\n", TrapFrame
->Dr7
);
46 DbgPrint("SegGs: %x\n", TrapFrame
->SegGs
);
47 DbgPrint("SegEs: %x\n", TrapFrame
->SegEs
);
48 DbgPrint("SegDs: %x\n", TrapFrame
->SegDs
);
49 DbgPrint("Edx: %x\n", TrapFrame
->Edx
);
50 DbgPrint("Ecx: %x\n", TrapFrame
->Ecx
);
51 DbgPrint("Eax: %x\n", TrapFrame
->Eax
);
52 DbgPrint("PreviousPreviousMode: %x\n", TrapFrame
->PreviousPreviousMode
);
53 DbgPrint("ExceptionList: %x\n", TrapFrame
->ExceptionList
);
54 DbgPrint("SegFs: %x\n", TrapFrame
->SegFs
);
55 DbgPrint("Edi: %x\n", TrapFrame
->Edi
);
56 DbgPrint("Esi: %x\n", TrapFrame
->Esi
);
57 DbgPrint("Ebx: %x\n", TrapFrame
->Ebx
);
58 DbgPrint("Ebp: %x\n", TrapFrame
->Ebp
);
59 DbgPrint("ErrCode: %x\n", TrapFrame
->ErrCode
);
60 DbgPrint("Eip: %x\n", TrapFrame
->Eip
);
61 DbgPrint("SegCs: %x\n", TrapFrame
->SegCs
);
62 DbgPrint("EFlags: %x\n", TrapFrame
->EFlags
);
63 DbgPrint("HardwareEsp: %x\n", TrapFrame
->HardwareEsp
);
64 DbgPrint("HardwareSegSs: %x\n", TrapFrame
->HardwareSegSs
);
65 DbgPrint("V86Es: %x\n", TrapFrame
->V86Es
);
66 DbgPrint("V86Ds: %x\n", TrapFrame
->V86Ds
);
67 DbgPrint("V86Fs: %x\n", TrapFrame
->V86Fs
);
68 DbgPrint("V86Gs: %x\n", TrapFrame
->V86Gs
);
74 KiFillTrapFrameDebug(IN PKTRAP_FRAME TrapFrame
)
76 /* Set the debug information */
77 TrapFrame
->DbgArgPointer
= TrapFrame
->Edx
;
78 TrapFrame
->DbgArgMark
= 0xBADB0D00;
79 TrapFrame
->DbgEip
= TrapFrame
->Eip
;
80 TrapFrame
->DbgEbp
= TrapFrame
->Ebp
;
85 KiExitTrapDebugChecks(IN PKTRAP_FRAME TrapFrame
,
86 IN KTRAP_STATE_BITS SkipBits
)
88 /* Make sure interrupts are disabled */
89 if (__readeflags() & EFLAGS_INTERRUPT_MASK
)
91 DbgPrint("Exiting with interrupts enabled: %lx\n", __readeflags());
95 /* Make sure this is a real trap frame */
96 if (TrapFrame
->DbgArgMark
!= 0xBADB0D00)
98 DbgPrint("Exiting with an invalid trap frame? (No MAGIC in trap frame)\n");
99 KiDumpTrapFrame(TrapFrame
);
103 /* Make sure we're not in user-mode or something */
104 if (Ke386GetFs() != KGDT_R0_PCR
)
106 DbgPrint("Exiting with an invalid FS: %lx\n", Ke386GetFs());
110 /* Make sure we have a valid SEH chain */
111 if (KeGetPcr()->Tib
.ExceptionList
== 0)
113 DbgPrint("Exiting with NULL exception chain: %p\n", KeGetPcr()->Tib
.ExceptionList
);
117 /* Make sure we're restoring a valid SEH chain */
118 if (TrapFrame
->ExceptionList
== 0)
120 DbgPrint("Entered a trap with a NULL exception chain: %p\n", TrapFrame
->ExceptionList
);
124 /* If we're ignoring previous mode, make sure caller doesn't actually want it */
125 if ((SkipBits
.SkipPreviousMode
) && (TrapFrame
->PreviousPreviousMode
!= -1))
127 DbgPrint("Exiting a trap witout restoring previous mode, yet previous mode seems valid: %lx", TrapFrame
->PreviousPreviousMode
);
134 KiExitSystemCallDebugChecks(IN ULONG SystemCall
,
135 IN PKTRAP_FRAME TrapFrame
)
139 /* Check if this was a user call */
140 if (KiUserMode(TrapFrame
))
142 /* Make sure we are not returning with elevated IRQL */
143 OldIrql
= KeGetCurrentIrql();
144 if (OldIrql
!= PASSIVE_LEVEL
)
146 /* Forcibly put us in a sane state */
147 KeGetPcr()->CurrentIrql
= PASSIVE_LEVEL
;
151 KeBugCheckEx(IRQL_GT_ZERO_AT_SYSTEM_SERVICE
,
158 /* Make sure we're not attached and that APCs are not disabled */
159 if ((KeGetCurrentThread()->ApcStateIndex
!= CurrentApcEnvironment
) ||
160 (KeGetCurrentThread()->CombinedApcDisable
!= 0))
163 KeBugCheckEx(APC_INDEX_MISMATCH
,
165 KeGetCurrentThread()->ApcStateIndex
,
166 KeGetCurrentThread()->CombinedApcDisable
,
172 #define KiExitTrapDebugChecks(x, y)
173 #define KiFillTrapFrameDebug(x)
174 #define KiExitSystemCallDebugChecks(x, y)
182 KiUserTrap(IN PKTRAP_FRAME TrapFrame
)
184 /* Anything else but Ring 0 is Ring 3 */
185 return (TrapFrame
->SegCs
& MODE_MASK
);
189 // Assembly exit stubs
193 /* Do not mark this as DECLSPEC_NORETURN because possibly executing code follows it! */
194 KiSystemCallReturn(IN PKTRAP_FRAME TrapFrame
)
196 /* Restore nonvolatiles, EAX, and do a "jump" back to the kernel caller */
200 "movl %c[b](%%esp), %%ebx\n"
201 "movl %c[s](%%esp), %%esi\n"
202 "movl %c[i](%%esp), %%edi\n"
203 "movl %c[p](%%esp), %%ebp\n"
204 "movl %c[a](%%esp), %%eax\n"
205 "movl %c[e](%%esp), %%edx\n"
206 "addl $%c[v],%%esp\n" /* A WHOLE *KERNEL* frame since we're not IRET'ing */
208 ".globl _KiSystemCallExit2\n_KiSystemCallExit2:\n"
211 [b
] "i"(KTRAP_FRAME_EBX
),
212 [s
] "i"(KTRAP_FRAME_ESI
),
213 [i
] "i"(KTRAP_FRAME_EDI
),
214 [p
] "i"(KTRAP_FRAME_EBP
),
215 [a
] "i"(KTRAP_FRAME_EAX
),
216 [e
] "i"(KTRAP_FRAME_EIP
),
217 [v
] "i"(KTRAP_FRAME_ESP
)
225 KiSystemCallTrapReturn(IN PKTRAP_FRAME TrapFrame
)
227 /* Regular interrupt exit, but we only restore EAX as a volatile */
230 ".globl _KiSystemCallExit\n_KiSystemCallExit:\n"
232 "movl %c[b](%%esp), %%ebx\n"
233 "movl %c[s](%%esp), %%esi\n"
234 "movl %c[i](%%esp), %%edi\n"
235 "movl %c[p](%%esp), %%ebp\n"
236 "movl %c[a](%%esp), %%eax\n"
237 "addl $%c[e],%%esp\n"
241 [b
] "i"(KTRAP_FRAME_EBX
),
242 [s
] "i"(KTRAP_FRAME_ESI
),
243 [i
] "i"(KTRAP_FRAME_EDI
),
244 [p
] "i"(KTRAP_FRAME_EBP
),
245 [a
] "i"(KTRAP_FRAME_EAX
),
246 [e
] "i"(KTRAP_FRAME_EIP
)
255 KiSystemCallSysExitReturn(IN PKTRAP_FRAME TrapFrame
)
257 /* Restore nonvolatiles, EAX, and do a SYSEXIT back to the user caller */
261 "movl %c[s](%%esp), %%esi\n"
262 "movl %c[b](%%esp), %%ebx\n"
263 "movl %c[i](%%esp), %%edi\n"
264 "movl %c[p](%%esp), %%ebp\n"
265 "movl %c[a](%%esp), %%eax\n"
266 "movl %c[e](%%esp), %%edx\n" /* SYSEXIT says EIP in EDX */
267 "movl %c[x](%%esp), %%ecx\n" /* SYSEXIT says ESP in ECX */
268 "addl $%c[v],%%esp\n" /* A WHOLE *USER* frame since we're not IRET'ing */
272 [b
] "i"(KTRAP_FRAME_EBX
),
273 [s
] "i"(KTRAP_FRAME_ESI
),
274 [i
] "i"(KTRAP_FRAME_EDI
),
275 [p
] "i"(KTRAP_FRAME_EBP
),
276 [a
] "i"(KTRAP_FRAME_EAX
),
277 [e
] "i"(KTRAP_FRAME_EIP
),
278 [x
] "i"(KTRAP_FRAME_ESP
),
279 [v
] "i"(KTRAP_FRAME_V86_ES
)
288 KiTrapReturn(IN PKTRAP_FRAME TrapFrame
)
290 /* Regular interrupt exit */
294 "movl %c[a](%%esp), %%eax\n"
295 "movl %c[b](%%esp), %%ebx\n"
296 "movl %c[c](%%esp), %%ecx\n"
297 "movl %c[d](%%esp), %%edx\n"
298 "movl %c[s](%%esp), %%esi\n"
299 "movl %c[i](%%esp), %%edi\n"
300 "movl %c[p](%%esp), %%ebp\n"
301 "addl $%c[e],%%esp\n"
305 [a
] "i"(KTRAP_FRAME_EAX
),
306 [b
] "i"(KTRAP_FRAME_EBX
),
307 [c
] "i"(KTRAP_FRAME_ECX
),
308 [d
] "i"(KTRAP_FRAME_EDX
),
309 [s
] "i"(KTRAP_FRAME_ESI
),
310 [i
] "i"(KTRAP_FRAME_EDI
),
311 [p
] "i"(KTRAP_FRAME_EBP
),
312 [e
] "i"(KTRAP_FRAME_EIP
)
321 KiDirectTrapReturn(IN PKTRAP_FRAME TrapFrame
)
323 /* Regular interrupt exit but we're not restoring any registers */
327 "addl $%c[e],%%esp\n"
331 [e
] "i"(KTRAP_FRAME_EIP
)
340 KiCallReturn(IN PKTRAP_FRAME TrapFrame
)
342 /* Pops a trap frame out of the stack but returns with RET instead of IRET */
346 "movl %c[b](%%esp), %%ebx\n"
347 "movl %c[s](%%esp), %%esi\n"
348 "movl %c[i](%%esp), %%edi\n"
349 "movl %c[p](%%esp), %%ebp\n"
350 "addl $%c[e],%%esp\n"
354 [b
] "i"(KTRAP_FRAME_EBX
),
355 [s
] "i"(KTRAP_FRAME_ESI
),
356 [i
] "i"(KTRAP_FRAME_EDI
),
357 [p
] "i"(KTRAP_FRAME_EBP
),
358 [e
] "i"(KTRAP_FRAME_EIP
)
367 KiEditedTrapReturn(IN PKTRAP_FRAME TrapFrame
)
369 /* Regular interrupt exit */
373 "movl %c[a](%%esp), %%eax\n"
374 "movl %c[b](%%esp), %%ebx\n"
375 "movl %c[c](%%esp), %%ecx\n"
376 "movl %c[d](%%esp), %%edx\n"
377 "movl %c[s](%%esp), %%esi\n"
378 "movl %c[i](%%esp), %%edi\n"
379 "movl %c[p](%%esp), %%ebp\n"
380 "addl $%c[e],%%esp\n"
381 "movl (%%esp), %%esp\n"
385 [a
] "i"(KTRAP_FRAME_EAX
),
386 [b
] "i"(KTRAP_FRAME_EBX
),
387 [c
] "i"(KTRAP_FRAME_ECX
),
388 [d
] "i"(KTRAP_FRAME_EDX
),
389 [s
] "i"(KTRAP_FRAME_ESI
),
390 [i
] "i"(KTRAP_FRAME_EDI
),
391 [p
] "i"(KTRAP_FRAME_EBP
),
392 [e
] "i"(KTRAP_FRAME_ERROR_CODE
) /* We *WANT* the error code since ESP is there! */
399 // "BOP" code used by VDM and V8086 Mode
405 /* Invalid instruction that an invalid opcode handler must trap and handle */
406 asm volatile(".byte 0xC4\n.byte 0xC4\n");
411 KiUserSystemCall(IN PKTRAP_FRAME TrapFrame
)
414 * Kernel call or user call?
416 * This decision is made in inlined assembly because we need to patch
417 * the relative offset of the user-mode jump to point to the SYSEXIT
418 * routine if the CPU supports it. The only way to guarantee that a
419 * relative jnz/jz instruction is generated is to force it with the
424 "test $1, %0\n" /* MODE_MASK */
425 ".globl _KiSystemCallExitBranch\n_KiSystemCallExitBranch:\n"
426 "jnz _KiSystemCallExit\n"
428 : "r"(TrapFrame
->SegCs
)
433 // Generic Exit Routine
438 KiExitTrap(IN PKTRAP_FRAME TrapFrame
,
441 KTRAP_EXIT_SKIP_BITS SkipBits
= { .Bits
= Skip
};
444 /* Debugging checks */
445 KiExitTrapDebugChecks(TrapFrame
, SkipBits
);
447 /* Restore the SEH handler chain */
448 KeGetPcr()->Tib
.ExceptionList
= TrapFrame
->ExceptionList
;
450 /* Check if the previous mode must be restored */
451 if (__builtin_expect(!SkipBits
.SkipPreviousMode
, 0)) /* More INTS than SYSCALLs */
454 KeGetCurrentThread()->PreviousMode
= TrapFrame
->PreviousPreviousMode
;
457 /* Check if there are active debug registers */
458 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
460 /* Not handled yet */
461 DbgPrint("Need Hardware Breakpoint Support!\n");
466 /* Check if this was a V8086 trap */
467 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0)) KiTrapReturn(TrapFrame
);
469 /* Check if the trap frame was edited */
470 if (__builtin_expect(!(TrapFrame
->SegCs
& FRAME_EDITED
), 0))
473 * An edited trap frame happens when we need to modify CS and/or ESP but
474 * don't actually have a ring transition. This happens when a kernelmode
475 * caller wants to perform an NtContinue to another kernel address, such
476 * as in the case of SEH (basically, a longjmp), or to a user address.
478 * Therefore, the CPU never saved CS/ESP on the stack because we did not
479 * get a trap frame due to a ring transition (there was no interrupt).
480 * Even if we didn't want to restore CS to a new value, a problem occurs
481 * due to the fact a normal RET would not work if we restored ESP since
482 * RET would then try to read the result off the stack.
484 * The NT kernel solves this by adding 12 bytes of stack to the exiting
485 * trap frame, in which EFLAGS, CS, and EIP are stored, and then saving
486 * the ESP that's being requested into the ErrorCode field. It will then
487 * exit with an IRET. This fixes both issues, because it gives the stack
488 * some space where to hold the return address and then end up with the
489 * wanted stack, and it uses IRET which allows a new CS to be inputted.
493 /* Set CS that is requested */
494 TrapFrame
->SegCs
= TrapFrame
->TempSegCs
;
496 /* First make space on requested stack */
497 ReturnStack
= (PULONG
)(TrapFrame
->TempEsp
- 12);
498 TrapFrame
->ErrCode
= (ULONG_PTR
)ReturnStack
;
500 /* Now copy IRET frame */
501 ReturnStack
[0] = TrapFrame
->Eip
;
502 ReturnStack
[1] = TrapFrame
->SegCs
;
503 ReturnStack
[2] = TrapFrame
->EFlags
;
505 /* Do special edited return */
506 KiEditedTrapReturn(TrapFrame
);
509 /* Check if this is a user trap */
510 if (__builtin_expect(KiUserTrap(TrapFrame
), 1)) /* Ring 3 is where we spend time */
512 /* Check if segments should be restored */
513 if (!SkipBits
.SkipSegments
)
515 /* Restore segments */
516 Ke386SetGs(TrapFrame
->SegGs
);
517 Ke386SetEs(TrapFrame
->SegEs
);
518 Ke386SetDs(TrapFrame
->SegDs
);
519 Ke386SetFs(TrapFrame
->SegFs
);
522 /* Always restore FS since it goes from KPCR to TEB */
523 Ke386SetFs(TrapFrame
->SegFs
);
526 /* Check for system call -- a system call skips volatiles! */
527 if (__builtin_expect(SkipBits
.SkipVolatiles
, 0)) /* More INTs than SYSCALLs */
529 /* User or kernel call? */
530 KiUserSystemCall(TrapFrame
);
533 __writeeflags(TrapFrame
->EFlags
);
535 /* Call is kernel, so do a jump back since this wasn't a real INT */
536 KiSystemCallReturn(TrapFrame
);
538 /* If we got here, this is SYSEXIT: are we stepping code? */
539 if (!(TrapFrame
->EFlags
& EFLAGS_TF
))
541 /* Restore user FS */
542 Ke386SetFs(KGDT_R3_TEB
| RPL_MASK
);
544 /* Remove interrupt flag */
545 TrapFrame
->EFlags
&= ~EFLAGS_INTERRUPT_MASK
;
546 __writeeflags(TrapFrame
->EFlags
);
548 /* Exit through SYSEXIT */
549 KiSystemCallSysExitReturn(TrapFrame
);
552 /* Exit through IRETD, either due to debugging or due to lack of SYSEXIT */
553 KiSystemCallTrapReturn(TrapFrame
);
556 /* Return from interrupt */
557 KiTrapReturn(TrapFrame
);
561 // Virtual 8086 Mode Optimized Trap Exit
565 KiExitV86Trap(IN PKTRAP_FRAME TrapFrame
)
571 Thread
= KeGetCurrentThread();
574 /* Turn off the alerted state for kernel mode */
575 Thread
->Alerted
[KernelMode
] = FALSE
;
577 /* Are there pending user APCs? */
578 if (__builtin_expect(!Thread
->ApcState
.UserApcPending
, 1)) break;
580 /* Raise to APC level and enable interrupts */
581 OldIrql
= KfRaiseIrql(APC_LEVEL
);
585 KiDeliverApc(UserMode
, NULL
, TrapFrame
);
587 /* Restore IRQL and disable interrupts once again */
588 KfLowerIrql(OldIrql
);
591 /* Return if this isn't V86 mode anymore */
592 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0)) return;
595 /* If we got here, we're still in a valid V8086 context, so quit it */
596 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
598 /* Not handled yet */
599 DbgPrint("Need Hardware Breakpoint Support!\n");
603 /* Return from interrupt */
604 KiTrapReturn(TrapFrame
);
608 // Virtual 8086 Mode Optimized Trap Entry
612 KiEnterV86Trap(IN PKTRAP_FRAME TrapFrame
)
614 /* Load correct registers */
615 Ke386SetFs(KGDT_R0_PCR
);
616 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
617 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
619 /* Save exception list and bogus previous mode */
620 TrapFrame
->PreviousPreviousMode
= -1;
621 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
623 /* Clear direction flag */
624 Ke386ClearDirectionFlag();
626 /* Save DR7 and check for debugging */
627 TrapFrame
->Dr7
= __readdr(7);
628 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
630 DbgPrint("Need Hardware Breakpoint Support!\n");
636 // Interrupt Trap Entry
640 KiEnterInterruptTrap(IN PKTRAP_FRAME TrapFrame
)
642 /* Set bogus previous mode */
643 TrapFrame
->PreviousPreviousMode
= -1;
645 /* Check for V86 mode */
646 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0))
648 /* Restore V8086 segments into Protected Mode segments */
649 TrapFrame
->SegFs
= TrapFrame
->V86Fs
;
650 TrapFrame
->SegGs
= TrapFrame
->V86Gs
;
651 TrapFrame
->SegDs
= TrapFrame
->V86Ds
;
652 TrapFrame
->SegEs
= TrapFrame
->V86Es
;
655 /* Check if this wasn't kernel code */
656 if (__builtin_expect(TrapFrame
->SegCs
!= KGDT_R0_CODE
, 1)) /* Ring 3 is more common */
658 /* Save segments and then switch to correct ones */
659 TrapFrame
->SegFs
= Ke386GetFs();
660 TrapFrame
->SegGs
= Ke386GetGs();
661 TrapFrame
->SegDs
= Ke386GetDs();
662 TrapFrame
->SegEs
= Ke386GetEs();
663 Ke386SetFs(KGDT_R0_PCR
);
664 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
665 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
668 /* Save exception list and terminate it */
669 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
670 KeGetPcr()->Tib
.ExceptionList
= EXCEPTION_CHAIN_END
;
673 TrapFrame
->ErrCode
= 0;
675 /* Clear direction flag */
676 Ke386ClearDirectionFlag();
678 /* Flush DR7 and check for debugging */
680 if (__builtin_expect(KeGetCurrentThread()->DispatcherHeader
.DebugActive
& 0xFF, 0))
682 DbgPrint("Need Hardware Breakpoint Support!\n");
686 /* Set debug header */
687 KiFillTrapFrameDebug(TrapFrame
);
691 // Generic Trap Entry
695 KiEnterTrap(IN PKTRAP_FRAME TrapFrame
)
700 * We really have to get a good DS/ES first before touching any data.
702 * These two reads will either go in a register (with optimizations ON) or
703 * a stack variable (which is on SS:ESP, guaranteed to be good/valid).
705 * Because the assembly is marked volatile, the order of instructions is
706 * as-is, otherwise the optimizer could simply get rid of our DS/ES.
711 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
712 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
713 TrapFrame
->SegDs
= Ds
;
714 TrapFrame
->SegEs
= Es
;
716 /* Now we can save the other segments and then switch to the correct FS */
717 TrapFrame
->SegFs
= Ke386GetFs();
718 TrapFrame
->SegGs
= Ke386GetGs();
719 Ke386SetFs(KGDT_R0_PCR
);
721 /* Save exception list and bogus previous mode */
722 TrapFrame
->PreviousPreviousMode
= -1;
723 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
725 /* Check for V86 mode */
726 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0))
728 /* Restore V8086 segments into Protected Mode segments */
729 TrapFrame
->SegFs
= TrapFrame
->V86Fs
;
730 TrapFrame
->SegGs
= TrapFrame
->V86Gs
;
731 TrapFrame
->SegDs
= TrapFrame
->V86Ds
;
732 TrapFrame
->SegEs
= TrapFrame
->V86Es
;
735 /* Clear direction flag */
736 Ke386ClearDirectionFlag();
738 /* Flush DR7 and check for debugging */
740 if (__builtin_expect(KeGetCurrentThread()->DispatcherHeader
.DebugActive
& 0xFF, 0))
742 DbgPrint("Need Hardware Breakpoint Support!\n");
746 /* Set debug header */
747 KiFillTrapFrameDebug(TrapFrame
);
751 // Generates a Trap Prolog Stub for the given name
753 #define KI_PUSH_FAKE_ERROR_CODE 0x1
754 #define KI_FAST_V86_TRAP 0x2
755 #define KI_NONVOLATILES_ONLY 0x4
756 #define KI_FAST_SYSTEM_CALL 0x8
757 #define KI_SOFTWARE_TRAP 0x10
758 #define KI_HARDWARE_INT 0x20
759 #define KiTrap(x, y) VOID DECLSPEC_NORETURN x(VOID) { KiTrapStub(y, x##Handler); UNREACHABLE; }
760 #define KiTrampoline(x, y) VOID DECLSPEC_NOINLINE x(VOID) { KiTrapStub(y, x##Handler); }
767 KiTrapStub(IN ULONG Flags
,
772 /* Is this a fast system call? They don't have a stack! */
773 if (Flags
& KI_FAST_SYSTEM_CALL
) __asm__ __volatile__
775 "movl %%ss:%c[t], %%esp\n"
776 "movl %c[e](%%esp), %%esp\n"
778 : [e
] "i"(FIELD_OFFSET(KTSS
, Esp0
)),
783 /* Check what kind of trap frame this trap requires */
784 if (Flags
& KI_SOFTWARE_TRAP
)
786 /* Software traps need a complete non-ring transition trap frame */
787 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, HardwareEsp
);
789 else if (Flags
& KI_FAST_SYSTEM_CALL
)
791 /* SYSENTER requires us to build a complete ring transition trap frame */
792 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, V86Es
);
794 /* And it only preserves nonvolatile registers */
795 Flags
|= KI_NONVOLATILES_ONLY
;
797 else if (Flags
& KI_PUSH_FAKE_ERROR_CODE
)
799 /* If the trap doesn't have an error code, we'll make space for it */
800 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, Eip
);
804 /* The trap already has an error code, so just make space for the rest */
805 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, ErrCode
);
808 /* Software traps need to get their EIP from the caller's frame */
809 if (Flags
& KI_SOFTWARE_TRAP
) __asm__
__volatile__ ("popl %%eax\n":::"%esp");
811 /* Now go ahead and make space for this frame */
812 __asm__
__volatile__ ("subl $%c[e],%%esp\n":: [e
] "i"(FrameSize
) : "%esp");
814 /* Does the caller want volatiles only? */
815 if (Flags
& KI_NONVOLATILES_ONLY
) __asm__ __volatile__
817 /* Then only EBX, ESI, EDI and EBP are saved */
818 "movl %%ebx, %c[b](%%esp)\n"
819 "movl %%esi, %c[s](%%esp)\n"
820 "movl %%edi, %c[i](%%esp)\n"
821 "movl %%ebp, %c[p](%%esp)\n"
823 : [b
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Ebx
)),
824 [s
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Esi
)),
825 [i
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Edi
)),
826 [p
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Ebp
))
829 else __asm__ __volatile__
831 /* Otherwise, we save all the registers (except ESP) */
832 "movl %%eax, %c[a](%%esp)\n"
833 "movl %%ebx, %c[b](%%esp)\n"
834 "movl %%ecx, %c[c](%%esp)\n"
835 "movl %%edx, %c[d](%%esp)\n"
836 "movl %%esi, %c[s](%%esp)\n"
837 "movl %%edi, %c[i](%%esp)\n"
838 "movl %%ebp, %c[p](%%esp)\n"
840 : [a
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Eax
)),
841 [b
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Ebx
)),
842 [c
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Ecx
)),
843 [d
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Edx
)),
844 [s
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Esi
)),
845 [i
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Edi
)),
846 [p
] "i"(FIELD_OFFSET(KTRAP_FRAME
, Ebp
))
850 /* Now set parameter 1 (ECX) to point to the frame */
851 __asm__
__volatile__ ("movl %%esp, %%ecx\n":::"%esp");
853 /* For Fast-V86 traps, set parameter 2 (EDX) to hold EFlags */
854 if (Flags
& KI_FAST_V86_TRAP
) __asm__ __volatile__
856 "movl %c[f](%%esp), %%edx\n"
858 : [f
] "i"(FIELD_OFFSET(KTRAP_FRAME
, EFlags
))
860 else if (Flags
& KI_HARDWARE_INT
) __asm__ __volatile__
863 * For hardware interrupts, set parameter 2 (EDX) to hold KINTERRUPT.
864 * This code will be dynamically patched when an interrupt is registered!
866 ".globl _KiInterruptTemplate2ndDispatch\n_KiInterruptTemplate2ndDispatch:\n"
868 ".globl _KiInterruptTemplateObject\n_KiInterruptTemplateObject:\n"
872 /* Now jump to the C handler */
873 if (Flags
& KI_HARDWARE_INT
)__asm__ __volatile__
876 * For hardware interrupts, use an absolute JMP instead of a relative JMP
877 * since the position of this code is arbitrary in memory, and therefore
878 * the compiler-generated offset will not be correct.
881 ".globl _KiInterruptTemplateDispatch\n_KiInterruptTemplateDispatch:\n"
885 else __asm__
__volatile__ ("jmp %c[x]\n":: [x
] "i"(Handler
));