2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/include/trap_x.h
5 * PURPOSE: Internal Inlined Functions for the Trap Handling Code
6 * PROGRAMMERS: ReactOS Portable Systems Group
12 // Unreachable code hint for GCC 4.5.x, older GCC versions, and MSVC
15 #if __GNUC__ * 100 + __GNUC_MINOR__ >= 405
16 #define UNREACHABLE __builtin_unreachable()
18 #define UNREACHABLE __builtin_trap()
21 #define UNREACHABLE __assume(0)
31 KiDumpTrapFrame(IN PKTRAP_FRAME TrapFrame
)
33 /* Dump the whole thing */
34 DbgPrint("DbgEbp: %x\n", TrapFrame
->DbgEbp
);
35 DbgPrint("DbgEip: %x\n", TrapFrame
->DbgEip
);
36 DbgPrint("DbgArgMark: %x\n", TrapFrame
->DbgArgMark
);
37 DbgPrint("DbgArgPointer: %x\n", TrapFrame
->DbgArgPointer
);
38 DbgPrint("TempSegCs: %x\n", TrapFrame
->TempSegCs
);
39 DbgPrint("TempEsp: %x\n", TrapFrame
->TempEsp
);
40 DbgPrint("Dr0: %x\n", TrapFrame
->Dr0
);
41 DbgPrint("Dr1: %x\n", TrapFrame
->Dr1
);
42 DbgPrint("Dr2: %x\n", TrapFrame
->Dr2
);
43 DbgPrint("Dr3: %x\n", TrapFrame
->Dr3
);
44 DbgPrint("Dr6: %x\n", TrapFrame
->Dr6
);
45 DbgPrint("Dr7: %x\n", TrapFrame
->Dr7
);
46 DbgPrint("SegGs: %x\n", TrapFrame
->SegGs
);
47 DbgPrint("SegEs: %x\n", TrapFrame
->SegEs
);
48 DbgPrint("SegDs: %x\n", TrapFrame
->SegDs
);
49 DbgPrint("Edx: %x\n", TrapFrame
->Edx
);
50 DbgPrint("Ecx: %x\n", TrapFrame
->Ecx
);
51 DbgPrint("Eax: %x\n", TrapFrame
->Eax
);
52 DbgPrint("PreviousPreviousMode: %x\n", TrapFrame
->PreviousPreviousMode
);
53 DbgPrint("ExceptionList: %x\n", TrapFrame
->ExceptionList
);
54 DbgPrint("SegFs: %x\n", TrapFrame
->SegFs
);
55 DbgPrint("Edi: %x\n", TrapFrame
->Edi
);
56 DbgPrint("Esi: %x\n", TrapFrame
->Esi
);
57 DbgPrint("Ebx: %x\n", TrapFrame
->Ebx
);
58 DbgPrint("Ebp: %x\n", TrapFrame
->Ebp
);
59 DbgPrint("ErrCode: %x\n", TrapFrame
->ErrCode
);
60 DbgPrint("Eip: %x\n", TrapFrame
->Eip
);
61 DbgPrint("SegCs: %x\n", TrapFrame
->SegCs
);
62 DbgPrint("EFlags: %x\n", TrapFrame
->EFlags
);
63 DbgPrint("HardwareEsp: %x\n", TrapFrame
->HardwareEsp
);
64 DbgPrint("HardwareSegSs: %x\n", TrapFrame
->HardwareSegSs
);
65 DbgPrint("V86Es: %x\n", TrapFrame
->V86Es
);
66 DbgPrint("V86Ds: %x\n", TrapFrame
->V86Ds
);
67 DbgPrint("V86Fs: %x\n", TrapFrame
->V86Fs
);
68 DbgPrint("V86Gs: %x\n", TrapFrame
->V86Gs
);
74 KiFillTrapFrameDebug(IN PKTRAP_FRAME TrapFrame
)
76 /* Set the debug information */
77 TrapFrame
->DbgArgPointer
= TrapFrame
->Edx
;
78 TrapFrame
->DbgArgMark
= 0xBADB0D00;
79 TrapFrame
->DbgEip
= TrapFrame
->Eip
;
80 TrapFrame
->DbgEbp
= TrapFrame
->Ebp
;
85 KiExitTrapDebugChecks(IN PKTRAP_FRAME TrapFrame
,
86 IN KTRAP_STATE_BITS SkipBits
)
88 /* Make sure interrupts are disabled */
89 if (__readeflags() & EFLAGS_INTERRUPT_MASK
)
91 DbgPrint("Exiting with interrupts enabled: %lx\n", __readeflags());
95 /* Make sure this is a real trap frame */
96 if (TrapFrame
->DbgArgMark
!= 0xBADB0D00)
98 DbgPrint("Exiting with an invalid trap frame? (No MAGIC in trap frame)\n");
99 KiDumpTrapFrame(TrapFrame
);
103 /* Make sure we're not in user-mode or something */
104 if (Ke386GetFs() != KGDT_R0_PCR
)
106 DbgPrint("Exiting with an invalid FS: %lx\n", Ke386GetFs());
110 /* Make sure we have a valid SEH chain */
111 if (KeGetPcr()->Tib
.ExceptionList
== 0)
113 DbgPrint("Exiting with NULL exception chain: %p\n", KeGetPcr()->Tib
.ExceptionList
);
117 /* Make sure we're restoring a valid SEH chain */
118 if (TrapFrame
->ExceptionList
== 0)
120 DbgPrint("Entered a trap with a NULL exception chain: %p\n", TrapFrame
->ExceptionList
);
124 /* If we're ignoring previous mode, make sure caller doesn't actually want it */
125 if ((SkipBits
.SkipPreviousMode
) && (TrapFrame
->PreviousPreviousMode
!= -1))
127 DbgPrint("Exiting a trap witout restoring previous mode, yet previous mode seems valid: %lx", TrapFrame
->PreviousPreviousMode
);
134 KiExitSystemCallDebugChecks(IN ULONG SystemCall
,
135 IN PKTRAP_FRAME TrapFrame
)
139 /* Check if this was a user call */
140 if (KiUserMode(TrapFrame
))
142 /* Make sure we are not returning with elevated IRQL */
143 OldIrql
= KeGetCurrentIrql();
144 if (OldIrql
!= PASSIVE_LEVEL
)
146 /* Forcibly put us in a sane state */
147 KeGetPcr()->CurrentIrql
= PASSIVE_LEVEL
;
151 KeBugCheckEx(IRQL_GT_ZERO_AT_SYSTEM_SERVICE
,
158 /* Make sure we're not attached and that APCs are not disabled */
159 if ((KeGetCurrentThread()->ApcStateIndex
!= CurrentApcEnvironment
) ||
160 (KeGetCurrentThread()->CombinedApcDisable
!= 0))
163 KeBugCheckEx(APC_INDEX_MISMATCH
,
165 KeGetCurrentThread()->ApcStateIndex
,
166 KeGetCurrentThread()->CombinedApcDisable
,
172 #define KiExitTrapDebugChecks(x, y)
173 #define KiFillTrapFrameDebug(x)
174 #define KiExitSystemCallDebugChecks(x, y)
182 KiUserTrap(IN PKTRAP_FRAME TrapFrame
)
184 /* Anything else but Ring 0 is Ring 3 */
185 return (TrapFrame
->SegCs
& MODE_MASK
);
189 // Assembly exit stubs
193 /* Do not mark this as DECLSPEC_NORETURN because possibly executing code follows it! */
194 KiSystemCallReturn(IN PKTRAP_FRAME TrapFrame
)
196 /* Restore nonvolatiles, EAX, and do a "jump" back to the kernel caller */
200 "movl %c[b](%%esp), %%ebx\n"
201 "movl %c[s](%%esp), %%esi\n"
202 "movl %c[i](%%esp), %%edi\n"
203 "movl %c[p](%%esp), %%ebp\n"
204 "movl %c[a](%%esp), %%eax\n"
205 "movl %c[e](%%esp), %%edx\n"
206 "addl $%c[v],%%esp\n" /* A WHOLE *KERNEL* frame since we're not IRET'ing */
208 ".globl _KiSystemCallExit2\n_KiSystemCallExit2:\n"
211 [b
] "i"(KTRAP_FRAME_EBX
),
212 [s
] "i"(KTRAP_FRAME_ESI
),
213 [i
] "i"(KTRAP_FRAME_EDI
),
214 [p
] "i"(KTRAP_FRAME_EBP
),
215 [a
] "i"(KTRAP_FRAME_EAX
),
216 [e
] "i"(KTRAP_FRAME_EIP
),
217 [v
] "i"(KTRAP_FRAME_ESP
)
225 KiSystemCallTrapReturn(IN PKTRAP_FRAME TrapFrame
)
227 /* Regular interrupt exit, but we only restore EAX as a volatile */
230 ".globl _KiSystemCallExit\n_KiSystemCallExit:\n"
232 "movl %c[b](%%esp), %%ebx\n"
233 "movl %c[s](%%esp), %%esi\n"
234 "movl %c[i](%%esp), %%edi\n"
235 "movl %c[p](%%esp), %%ebp\n"
236 "movl %c[a](%%esp), %%eax\n"
237 "addl $%c[e],%%esp\n"
241 [b
] "i"(KTRAP_FRAME_EBX
),
242 [s
] "i"(KTRAP_FRAME_ESI
),
243 [i
] "i"(KTRAP_FRAME_EDI
),
244 [p
] "i"(KTRAP_FRAME_EBP
),
245 [a
] "i"(KTRAP_FRAME_EAX
),
246 [e
] "i"(KTRAP_FRAME_EIP
)
255 KiSystemCallSysExitReturn(IN PKTRAP_FRAME TrapFrame
)
257 /* Restore nonvolatiles, EAX, and do a SYSEXIT back to the user caller */
261 "movl %c[s](%%esp), %%esi\n"
262 "movl %c[b](%%esp), %%ebx\n"
263 "movl %c[i](%%esp), %%edi\n"
264 "movl %c[p](%%esp), %%ebp\n"
265 "movl %c[a](%%esp), %%eax\n"
266 "movl %c[e](%%esp), %%edx\n" /* SYSEXIT says EIP in EDX */
267 "movl %c[x](%%esp), %%ecx\n" /* SYSEXIT says ESP in ECX */
268 "addl $%c[v],%%esp\n" /* A WHOLE *USER* frame since we're not IRET'ing */
272 [b
] "i"(KTRAP_FRAME_EBX
),
273 [s
] "i"(KTRAP_FRAME_ESI
),
274 [i
] "i"(KTRAP_FRAME_EDI
),
275 [p
] "i"(KTRAP_FRAME_EBP
),
276 [a
] "i"(KTRAP_FRAME_EAX
),
277 [e
] "i"(KTRAP_FRAME_EIP
),
278 [x
] "i"(KTRAP_FRAME_ESP
),
279 [v
] "i"(KTRAP_FRAME_V86_ES
)
288 KiTrapReturn(IN PKTRAP_FRAME TrapFrame
)
290 /* Regular interrupt exit */
294 "movl %c[a](%%esp), %%eax\n"
295 "movl %c[b](%%esp), %%ebx\n"
296 "movl %c[c](%%esp), %%ecx\n"
297 "movl %c[d](%%esp), %%edx\n"
298 "movl %c[s](%%esp), %%esi\n"
299 "movl %c[i](%%esp), %%edi\n"
300 "movl %c[p](%%esp), %%ebp\n"
301 "addl $%c[e],%%esp\n"
305 [a
] "i"(KTRAP_FRAME_EAX
),
306 [b
] "i"(KTRAP_FRAME_EBX
),
307 [c
] "i"(KTRAP_FRAME_ECX
),
308 [d
] "i"(KTRAP_FRAME_EDX
),
309 [s
] "i"(KTRAP_FRAME_ESI
),
310 [i
] "i"(KTRAP_FRAME_EDI
),
311 [p
] "i"(KTRAP_FRAME_EBP
),
312 [e
] "i"(KTRAP_FRAME_EIP
)
321 KiDirectTrapReturn(IN PKTRAP_FRAME TrapFrame
)
323 /* Regular interrupt exit but we're not restoring any registers */
327 "addl $%c[e],%%esp\n"
331 [e
] "i"(KTRAP_FRAME_EIP
)
340 KiCallReturn(IN PKTRAP_FRAME TrapFrame
)
342 /* Pops a trap frame out of the stack but returns with RET instead of IRET */
346 "movl %c[b](%%esp), %%ebx\n"
347 "movl %c[s](%%esp), %%esi\n"
348 "movl %c[i](%%esp), %%edi\n"
349 "movl %c[p](%%esp), %%ebp\n"
350 "addl $%c[e],%%esp\n"
354 [b
] "i"(KTRAP_FRAME_EBX
),
355 [s
] "i"(KTRAP_FRAME_ESI
),
356 [i
] "i"(KTRAP_FRAME_EDI
),
357 [p
] "i"(KTRAP_FRAME_EBP
),
358 [e
] "i"(KTRAP_FRAME_EIP
)
367 KiEditedTrapReturn(IN PKTRAP_FRAME TrapFrame
)
369 /* Regular interrupt exit */
373 "movl %c[a](%%esp), %%eax\n"
374 "movl %c[b](%%esp), %%ebx\n"
375 "movl %c[c](%%esp), %%ecx\n"
376 "movl %c[d](%%esp), %%edx\n"
377 "movl %c[s](%%esp), %%esi\n"
378 "movl %c[i](%%esp), %%edi\n"
379 "movl %c[p](%%esp), %%ebp\n"
380 "addl $%c[e],%%esp\n"
381 "movl (%%esp), %%esp\n"
385 [a
] "i"(KTRAP_FRAME_EAX
),
386 [b
] "i"(KTRAP_FRAME_EBX
),
387 [c
] "i"(KTRAP_FRAME_ECX
),
388 [d
] "i"(KTRAP_FRAME_EDX
),
389 [s
] "i"(KTRAP_FRAME_ESI
),
390 [i
] "i"(KTRAP_FRAME_EDI
),
391 [p
] "i"(KTRAP_FRAME_EBP
),
392 [e
] "i"(KTRAP_FRAME_ERROR_CODE
) /* We *WANT* the error code since ESP is there! */
399 // "BOP" code used by VDM and V8086 Mode
405 /* Invalid instruction that an invalid opcode handler must trap and handle */
406 asm volatile(".byte 0xC4\n.byte 0xC4\n");
411 KiUserSystemCall(IN PKTRAP_FRAME TrapFrame
)
414 * Kernel call or user call?
416 * This decision is made in inlined assembly because we need to patch
417 * the relative offset of the user-mode jump to point to the SYSEXIT
418 * routine if the CPU supports it. The only way to guarantee that a
419 * relative jnz/jz instruction is generated is to force it with the
424 "test $1, %0\n" /* MODE_MASK */
425 ".globl _KiSystemCallExitBranch\n_KiSystemCallExitBranch:\n"
426 "jnz _KiSystemCallExit\n"
428 : "r"(TrapFrame
->SegCs
)
433 // Generic Exit Routine
438 KiExitTrap(IN PKTRAP_FRAME TrapFrame
,
441 KTRAP_EXIT_SKIP_BITS SkipBits
= { .Bits
= Skip
};
444 /* Debugging checks */
445 KiExitTrapDebugChecks(TrapFrame
, SkipBits
);
447 /* Restore the SEH handler chain */
448 KeGetPcr()->Tib
.ExceptionList
= TrapFrame
->ExceptionList
;
450 /* Check if the previous mode must be restored */
451 if (__builtin_expect(!SkipBits
.SkipPreviousMode
, 0)) /* More INTS than SYSCALLs */
454 KeGetCurrentThread()->PreviousMode
= TrapFrame
->PreviousPreviousMode
;
457 /* Check if there are active debug registers */
458 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
460 /* Not handled yet */
461 DbgPrint("Need Hardware Breakpoint Support!\n");
466 /* Check if this was a V8086 trap */
467 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0)) KiTrapReturn(TrapFrame
);
469 /* Check if the trap frame was edited */
470 if (__builtin_expect(!(TrapFrame
->SegCs
& FRAME_EDITED
), 0))
473 * An edited trap frame happens when we need to modify CS and/or ESP but
474 * don't actually have a ring transition. This happens when a kernelmode
475 * caller wants to perform an NtContinue to another kernel address, such
476 * as in the case of SEH (basically, a longjmp), or to a user address.
478 * Therefore, the CPU never saved CS/ESP on the stack because we did not
479 * get a trap frame due to a ring transition (there was no interrupt).
480 * Even if we didn't want to restore CS to a new value, a problem occurs
481 * due to the fact a normal RET would not work if we restored ESP since
482 * RET would then try to read the result off the stack.
484 * The NT kernel solves this by adding 12 bytes of stack to the exiting
485 * trap frame, in which EFLAGS, CS, and EIP are stored, and then saving
486 * the ESP that's being requested into the ErrorCode field. It will then
487 * exit with an IRET. This fixes both issues, because it gives the stack
488 * some space where to hold the return address and then end up with the
489 * wanted stack, and it uses IRET which allows a new CS to be inputted.
493 /* Set CS that is requested */
494 TrapFrame
->SegCs
= TrapFrame
->TempSegCs
;
496 /* First make space on requested stack */
497 ReturnStack
= (PULONG
)(TrapFrame
->TempEsp
- 12);
498 TrapFrame
->ErrCode
= (ULONG_PTR
)ReturnStack
;
500 /* Now copy IRET frame */
501 ReturnStack
[0] = TrapFrame
->Eip
;
502 ReturnStack
[1] = TrapFrame
->SegCs
;
503 ReturnStack
[2] = TrapFrame
->EFlags
;
505 /* Do special edited return */
506 KiEditedTrapReturn(TrapFrame
);
509 /* Check if this is a user trap */
510 if (__builtin_expect(KiUserTrap(TrapFrame
), 1)) /* Ring 3 is where we spend time */
512 /* Check if segments should be restored */
513 if (!SkipBits
.SkipSegments
)
515 /* Restore segments */
516 Ke386SetGs(TrapFrame
->SegGs
);
517 Ke386SetEs(TrapFrame
->SegEs
);
518 Ke386SetDs(TrapFrame
->SegDs
);
519 Ke386SetFs(TrapFrame
->SegFs
);
522 /* Always restore FS since it goes from KPCR to TEB */
523 Ke386SetFs(TrapFrame
->SegFs
);
526 /* Check for system call -- a system call skips volatiles! */
527 if (__builtin_expect(SkipBits
.SkipVolatiles
, 0)) /* More INTs than SYSCALLs */
529 /* User or kernel call? */
530 KiUserSystemCall(TrapFrame
);
533 __writeeflags(TrapFrame
->EFlags
);
535 /* Call is kernel, so do a jump back since this wasn't a real INT */
536 KiSystemCallReturn(TrapFrame
);
538 /* If we got here, this is SYSEXIT: are we stepping code? */
539 if (!(TrapFrame
->EFlags
& EFLAGS_TF
))
541 /* Restore user FS */
542 Ke386SetFs(KGDT_R3_TEB
| RPL_MASK
);
544 /* Remove interrupt flag */
545 TrapFrame
->EFlags
&= ~EFLAGS_INTERRUPT_MASK
;
546 __writeeflags(TrapFrame
->EFlags
);
548 /* Exit through SYSEXIT */
549 KiSystemCallSysExitReturn(TrapFrame
);
552 /* Exit through IRETD, either due to debugging or due to lack of SYSEXIT */
553 KiSystemCallTrapReturn(TrapFrame
);
556 /* Return from interrupt */
557 KiTrapReturn(TrapFrame
);
561 // Virtual 8086 Mode Optimized Trap Exit
565 KiExitV86Trap(IN PKTRAP_FRAME TrapFrame
)
571 Thread
= KeGetCurrentThread();
574 /* Turn off the alerted state for kernel mode */
575 Thread
->Alerted
[KernelMode
] = FALSE
;
577 /* Are there pending user APCs? */
578 if (__builtin_expect(!Thread
->ApcState
.UserApcPending
, 1)) break;
580 /* Raise to APC level and enable interrupts */
581 OldIrql
= KfRaiseIrql(APC_LEVEL
);
585 KiDeliverApc(UserMode
, NULL
, TrapFrame
);
587 /* Restore IRQL and disable interrupts once again */
588 KfLowerIrql(OldIrql
);
591 /* Return if this isn't V86 mode anymore */
592 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0)) return;
595 /* If we got here, we're still in a valid V8086 context, so quit it */
596 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
598 /* Not handled yet */
599 DbgPrint("Need Hardware Breakpoint Support!\n");
603 /* Return from interrupt */
604 KiTrapReturn(TrapFrame
);
608 // Virtual 8086 Mode Optimized Trap Entry
612 KiEnterV86Trap(IN PKTRAP_FRAME TrapFrame
)
614 /* Load correct registers */
615 Ke386SetFs(KGDT_R0_PCR
);
616 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
617 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
619 /* Save exception list */
620 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
622 /* Clear direction flag */
623 Ke386ClearDirectionFlag();
625 /* Save DR7 and check for debugging */
626 TrapFrame
->Dr7
= __readdr(7);
627 if (__builtin_expect(TrapFrame
->Dr7
& ~DR7_RESERVED_MASK
, 0))
629 DbgPrint("Need Hardware Breakpoint Support!\n");
635 // Interrupt Trap Entry
639 KiEnterInterruptTrap(IN PKTRAP_FRAME TrapFrame
)
641 /* Check for V86 mode, otherwise check for ring 3 code */
642 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0))
644 /* Restore V8086 segments into Protected Mode segments */
645 TrapFrame
->SegFs
= TrapFrame
->V86Fs
;
646 TrapFrame
->SegGs
= TrapFrame
->V86Gs
;
647 TrapFrame
->SegDs
= TrapFrame
->V86Ds
;
648 TrapFrame
->SegEs
= TrapFrame
->V86Es
;
650 else if (__builtin_expect(TrapFrame
->SegCs
!= KGDT_R0_CODE
, 1)) /* Ring 3 is more common */
652 /* Save segments and then switch to correct ones */
653 TrapFrame
->SegFs
= Ke386GetFs();
654 TrapFrame
->SegGs
= Ke386GetGs();
655 TrapFrame
->SegDs
= Ke386GetDs();
656 TrapFrame
->SegEs
= Ke386GetEs();
659 /* Set correct segments */
660 Ke386SetFs(KGDT_R0_PCR
);
661 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
662 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
664 /* Save exception list and terminate it */
665 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
666 KeGetPcr()->Tib
.ExceptionList
= EXCEPTION_CHAIN_END
;
668 /* Clear direction flag */
669 Ke386ClearDirectionFlag();
671 /* Flush DR7 and check for debugging */
673 if (__builtin_expect(KeGetCurrentThread()->DispatcherHeader
.DebugActive
& 0xFF, 0))
675 DbgPrint("Need Hardware Breakpoint Support!\n");
679 /* Set debug header */
680 KiFillTrapFrameDebug(TrapFrame
);
684 // Generic Trap Entry
688 KiEnterTrap(IN PKTRAP_FRAME TrapFrame
)
693 * We really have to get a good DS/ES first before touching any data.
695 * These two reads will either go in a register (with optimizations ON) or
696 * a stack variable (which is on SS:ESP, guaranteed to be good/valid).
698 * Because the assembly is marked volatile, the order of instructions is
699 * as-is, otherwise the optimizer could simply get rid of our DS/ES.
704 Ke386SetDs(KGDT_R3_DATA
| RPL_MASK
);
705 Ke386SetEs(KGDT_R3_DATA
| RPL_MASK
);
706 TrapFrame
->SegDs
= Ds
;
707 TrapFrame
->SegEs
= Es
;
709 /* Now we can save the other segments and then switch to the correct FS */
710 TrapFrame
->SegFs
= Ke386GetFs();
711 TrapFrame
->SegGs
= Ke386GetGs();
712 Ke386SetFs(KGDT_R0_PCR
);
714 /* Save exception list */
715 TrapFrame
->ExceptionList
= KeGetPcr()->Tib
.ExceptionList
;
717 /* Check for V86 mode */
718 if (__builtin_expect(TrapFrame
->EFlags
& EFLAGS_V86_MASK
, 0))
720 /* Restore V8086 segments into Protected Mode segments */
721 TrapFrame
->SegFs
= TrapFrame
->V86Fs
;
722 TrapFrame
->SegGs
= TrapFrame
->V86Gs
;
723 TrapFrame
->SegDs
= TrapFrame
->V86Ds
;
724 TrapFrame
->SegEs
= TrapFrame
->V86Es
;
727 /* Clear direction flag */
728 Ke386ClearDirectionFlag();
730 /* Flush DR7 and check for debugging */
732 if (__builtin_expect(KeGetCurrentThread()->DispatcherHeader
.DebugActive
& 0xFF, 0))
734 DbgPrint("Need Hardware Breakpoint Support!\n");
738 /* Set debug header */
739 KiFillTrapFrameDebug(TrapFrame
);
743 // Generates a Trap Prolog Stub for the given name
745 #define KI_PUSH_FAKE_ERROR_CODE 0x1
746 #define KI_FAST_V86_TRAP 0x2
747 #define KI_NONVOLATILES_ONLY 0x4
748 #define KI_FAST_SYSTEM_CALL 0x8
749 #define KI_SOFTWARE_TRAP 0x10
750 #define KI_HARDWARE_INT 0x20
751 #define KiTrap(x, y) VOID DECLSPEC_NORETURN x(VOID) { KiTrapStub(y, x##Handler); UNREACHABLE; }
752 #define KiTrampoline(x, y) VOID DECLSPEC_NOINLINE x(VOID) { KiTrapStub(y, x##Handler); }
759 KiTrapStub(IN ULONG Flags
,
764 /* Is this a fast system call? They don't have a stack! */
765 if (Flags
& KI_FAST_SYSTEM_CALL
) __asm__ __volatile__
767 "movl %%ss:%c[t], %%esp\n"
768 "movl %c[e](%%esp), %%esp\n"
770 : [e
] "i"(FIELD_OFFSET(KTSS
, Esp0
)),
775 /* Check what kind of trap frame this trap requires */
776 if (Flags
& KI_SOFTWARE_TRAP
)
778 /* Software traps need a complete non-ring transition trap frame */
779 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, HardwareEsp
);
781 else if (Flags
& KI_FAST_SYSTEM_CALL
)
783 /* SYSENTER requires us to build a complete ring transition trap frame */
784 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, V86Es
);
786 /* And it only preserves nonvolatile registers */
787 Flags
|= KI_NONVOLATILES_ONLY
;
789 else if (Flags
& KI_PUSH_FAKE_ERROR_CODE
)
791 /* If the trap doesn't have an error code, we'll make space for it */
792 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, Eip
);
796 /* The trap already has an error code, so just make space for the rest */
797 FrameSize
= FIELD_OFFSET(KTRAP_FRAME
, ErrCode
);
800 /* Software traps need to get their EIP from the caller's frame */
801 if (Flags
& KI_SOFTWARE_TRAP
) __asm__
__volatile__ ("popl %%eax\n":::"%esp");
803 /* Save nonvolatile registers */
806 /* EBX, ESI, EDI and EBP are saved */
807 "movl %%ebp, %c[p](%%esp)\n"
808 "movl %%ebx, %c[b](%%esp)\n"
809 "movl %%esi, %c[s](%%esp)\n"
810 "movl %%edi, %c[i](%%esp)\n"
812 : [b
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Ebx
)),
813 [s
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Esi
)),
814 [i
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Edi
)),
815 [p
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Ebp
))
819 /* Does the caller want nonvolatiles only? */
820 if (!(Flags
& KI_NONVOLATILES_ONLY
)) __asm__ __volatile__
822 /* Otherwise, save the volatiles as well */
823 "movl %%eax, %c[a](%%esp)\n"
824 "movl %%ecx, %c[c](%%esp)\n"
825 "movl %%edx, %c[d](%%esp)\n"
827 : [a
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Eax
)),
828 [c
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Ecx
)),
829 [d
] "i"(- FrameSize
+ FIELD_OFFSET(KTRAP_FRAME
, Edx
))
833 /* Now set parameter 1 (ECX) to point to the frame */
834 __asm__
__volatile__ ("movl %%esp, %%ecx\n":::"%esp");
836 /* Now go ahead and make space for this frame */
837 __asm__
__volatile__ ("subl $%c[e],%%esp\n":: [e
] "i"(FrameSize
) : "%esp");
838 __asm__
__volatile__ ("subl $%c[e],%%ecx\n":: [e
] "i"(FrameSize
) : "%ecx");
840 /* For Fast-V86 traps, set parameter 2 (EDX) to hold EFlags */
841 if (Flags
& KI_FAST_V86_TRAP
) __asm__ __volatile__
843 "movl %c[f](%%esp), %%edx\n"
845 : [f
] "i"(FIELD_OFFSET(KTRAP_FRAME
, EFlags
))
847 else if (Flags
& KI_HARDWARE_INT
) __asm__ __volatile__
850 * For hardware interrupts, set parameter 2 (EDX) to hold KINTERRUPT.
851 * This code will be dynamically patched when an interrupt is registered!
853 ".globl _KiInterruptTemplate2ndDispatch\n_KiInterruptTemplate2ndDispatch:\n"
855 ".globl _KiInterruptTemplateObject\n_KiInterruptTemplateObject:\n"
859 /* Now jump to the C handler */
860 if (Flags
& KI_HARDWARE_INT
)__asm__ __volatile__
863 * For hardware interrupts, use an absolute JMP instead of a relative JMP
864 * since the position of this code is arbitrary in memory, and therefore
865 * the compiler-generated offset will not be correct.
868 ".globl _KiInterruptTemplateDispatch\n_KiInterruptTemplateDispatch:\n"
872 else __asm__
__volatile__ ("jmp %c[x]\n":: [x
] "i"(Handler
));