10eb6eb1bed9f0ba5faf6f00915641cf142d939d
[reactos.git] / ntoskrnl / include / internal / i386 / ke.h
1 #pragma once
2
3 #ifndef __ASM__
4
5 #include "intrin_i.h"
6
7 //
8 // Thread Dispatcher Header DebugActive Mask
9 //
10 #define DR_MASK(x) (1 << (x))
11 #define DR_REG_MASK 0x4F
12
13 //
14 // INT3 is 1 byte long
15 //
16 #define KD_BREAKPOINT_TYPE UCHAR
17 #define KD_BREAKPOINT_SIZE sizeof(UCHAR)
18 #define KD_BREAKPOINT_VALUE 0xCC
19
20 //
21 // Macros for getting and setting special purpose registers in portable code
22 //
23 #define KeGetContextPc(Context) \
24 ((Context)->Eip)
25
26 #define KeSetContextPc(Context, ProgramCounter) \
27 ((Context)->Eip = (ProgramCounter))
28
29 #define KeGetTrapFramePc(TrapFrame) \
30 ((TrapFrame)->Eip)
31
32 #define KiGetLinkedTrapFrame(x) \
33 (PKTRAP_FRAME)((x)->Edx)
34
35 #define KeGetContextReturnRegister(Context) \
36 ((Context)->Eax)
37
38 #define KeSetContextReturnRegister(Context, ReturnValue) \
39 ((Context)->Eax = (ReturnValue))
40
41 //
42 // Macro to get trap and exception frame from a thread stack
43 //
44 #define KeGetTrapFrame(Thread) \
45 (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
46 sizeof(KTRAP_FRAME) - \
47 sizeof(FX_SAVE_AREA))
48
49 #define KeGetExceptionFrame(Thread) \
50 NULL
51
52 //
53 // Macro to get context switches from the PRCB
54 // All architectures but x86 have it in the PRCB's KeContextSwitches
55 //
56 #define KeGetContextSwitches(Prcb) \
57 CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
58
59 //
60 // Macro to get the second level cache size field name which differs between
61 // CISC and RISC architectures, as the former has unified I/D cache
62 //
63 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
64
65 //
66 // Returns the Interrupt State from a Trap Frame.
67 // ON = TRUE, OFF = FALSE
68 //
69 #define KeGetTrapFrameInterruptState(TrapFrame) \
70 BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
71
72 //
73 // Flags for exiting a trap
74 //
75 #define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
76 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits)
77 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
78
79 typedef union _KTRAP_EXIT_SKIP_BITS
80 {
81 struct
82 {
83 UCHAR SkipPreviousMode:1;
84 UCHAR SkipSegments:1;
85 UCHAR SkipVolatiles:1;
86 UCHAR Reserved:5;
87 };
88 UCHAR Bits;
89 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
90
91
92 //
93 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
94 //
95 #define PFX_FLAG_ES 0x00000100
96 #define PFX_FLAG_CS 0x00000200
97 #define PFX_FLAG_SS 0x00000400
98 #define PFX_FLAG_DS 0x00000800
99 #define PFX_FLAG_FS 0x00001000
100 #define PFX_FLAG_GS 0x00002000
101 #define PFX_FLAG_OPER32 0x00004000
102 #define PFX_FLAG_ADDR32 0x00008000
103 #define PFX_FLAG_LOCK 0x00010000
104 #define PFX_FLAG_REPNE 0x00020000
105 #define PFX_FLAG_REP 0x00040000
106
107 //
108 // VDM Helper Macros
109 //
110 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
111 // We need to keep 2 parameters while the original ASM implementation uses 4:
112 // TrapFrame, PrefixFlags, Eip, InstructionSize;
113 //
114 // We pass the trap frame, and prefix flags, in our two parameters.
115 //
116 // We then realize that since the smallest prefix flag is 0x100, this gives us
117 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
118 //
119 // We further realize that we always have access to EIP from the trap frame, and
120 // that if we want the *current instruction* EIP, we simply have to add the
121 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
122 // at now, so we don't need to use the stack to push this parameter.
123 //
124 // We actually only care about the *current instruction* EIP in one location,
125 // so although it may be slightly more expensive to re-calculate the EIP one
126 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
127 // which would be forcing stack usage in all other scenarios.
128 //
129 #define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x));
130 #define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x))
131 #define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags)
132 #define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x)
133 #define KiVdmUnhandledOpcode(x) \
134 BOOLEAN \
135 FASTCALL \
136 KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \
137 IN ULONG Flags) \
138 { \
139 /* Not yet handled */ \
140 UNIMPLEMENTED_DBGBREAK(); \
141 return FALSE; \
142 }
143
144 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
145
146 //
147 // Local parameters
148 //
149 typedef struct _KV86_FRAME
150 {
151 PVOID ThreadStack;
152 PVOID ThreadTeb;
153 PVOID PcrTeb;
154 } KV86_FRAME, *PKV86_FRAME;
155
156 //
157 // Virtual Stack Frame
158 //
159 typedef struct _KV8086_STACK_FRAME
160 {
161 KTRAP_FRAME TrapFrame;
162 FX_SAVE_AREA NpxArea;
163 KV86_FRAME V86Frame;
164 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
165
166 //
167 // Large Pages Support
168 //
169 typedef struct _LARGE_IDENTITY_MAP
170 {
171 PHARDWARE_PTE TopLevelDirectory;
172 ULONG Cr3;
173 ULONG_PTR StartAddress;
174 ULONG PagesCount;
175 PVOID PagesList[30];
176 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
177
178 /* Diable interrupts and return whether they were enabled before */
179 FORCEINLINE
180 BOOLEAN
181 KeDisableInterrupts(VOID)
182 {
183 ULONG Flags;
184 BOOLEAN Return;
185
186 /* Get EFLAGS and check if the interrupt bit is set */
187 Flags = __readeflags();
188 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
189
190 /* Disable interrupts */
191 _disable();
192 return Return;
193 }
194
195 /* Restore previous interrupt state */
196 FORCEINLINE
197 VOID
198 KeRestoreInterrupts(BOOLEAN WereEnabled)
199 {
200 if (WereEnabled) _enable();
201 }
202
203 //
204 // Registers an interrupt handler with an IDT vector
205 //
206 FORCEINLINE
207 VOID
208 KeRegisterInterruptHandler(IN ULONG Vector,
209 IN PVOID Handler)
210 {
211 UCHAR Entry;
212 ULONG_PTR Address;
213 PKIPCR Pcr = (PKIPCR)KeGetPcr();
214
215 //
216 // Get the entry from the HAL
217 //
218 Entry = HalVectorToIDTEntry(Vector);
219 Address = PtrToUlong(Handler);
220
221 //
222 // Now set the data
223 //
224 Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
225 Pcr->IDT[Entry].Offset = (USHORT)Address;
226 }
227
228 //
229 // Returns the registered interrupt handler for a given IDT vector
230 //
231 FORCEINLINE
232 PVOID
233 KeQueryInterruptHandler(IN ULONG Vector)
234 {
235 PKIPCR Pcr = (PKIPCR)KeGetPcr();
236 UCHAR Entry;
237
238 //
239 // Get the entry from the HAL
240 //
241 Entry = HalVectorToIDTEntry(Vector);
242
243 //
244 // Read the entry from the IDT
245 //
246 return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
247 (Pcr->IDT[Entry].Offset & 0xFFFF));
248 }
249
250 //
251 // Invalidates the TLB entry for a specified address
252 //
253 FORCEINLINE
254 VOID
255 KeInvalidateTlbEntry(IN PVOID Address)
256 {
257 /* Invalidate the TLB entry for this address */
258 __invlpg(Address);
259 }
260
261 FORCEINLINE
262 VOID
263 KeFlushProcessTb(VOID)
264 {
265 /* Flush the TLB by resetting CR3 */
266 __writecr3(__readcr3());
267 }
268
269 FORCEINLINE
270 VOID
271 KeSweepICache(IN PVOID BaseAddress,
272 IN SIZE_T FlushSize)
273 {
274 //
275 // Always sweep the whole cache
276 //
277 UNREFERENCED_PARAMETER(BaseAddress);
278 UNREFERENCED_PARAMETER(FlushSize);
279 __wbinvd();
280 }
281
282 FORCEINLINE
283 PRKTHREAD
284 KeGetCurrentThread(VOID)
285 {
286 /* Return the current thread */
287 return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
288 }
289
290 FORCEINLINE
291 VOID
292 KiRundownThread(IN PKTHREAD Thread)
293 {
294 #ifndef CONFIG_SMP
295 /* Check if this is the NPX Thread */
296 if (KeGetCurrentPrcb()->NpxThread == Thread)
297 {
298 /* Clear it */
299 KeGetCurrentPrcb()->NpxThread = NULL;
300 Ke386FnInit();
301 }
302 #else
303 /* Nothing to do */
304 #endif
305 }
306
307 FORCEINLINE
308 VOID
309 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)
310 {
311 GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF);
312 GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16);
313 GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24);
314 }
315
316 FORCEINLINE
317 VOID
318 KiSetTebBase(PKPCR Pcr, PVOID TebAddress)
319 {
320 Pcr->NtTib.Self = TebAddress;
321 Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress);
322 }
323
324 INIT_FUNCTION
325 VOID
326 FASTCALL
327 Ki386InitializeTss(
328 IN PKTSS Tss,
329 IN PKIDTENTRY Idt,
330 IN PKGDTENTRY Gdt
331 );
332
333 INIT_FUNCTION
334 VOID
335 NTAPI
336 KiSetCR0Bits(VOID);
337
338 INIT_FUNCTION
339 VOID
340 NTAPI
341 KiGetCacheInformation(VOID);
342
343 INIT_FUNCTION
344 BOOLEAN
345 NTAPI
346 KiIsNpxPresent(
347 VOID
348 );
349
350 INIT_FUNCTION
351 BOOLEAN
352 NTAPI
353 KiIsNpxErrataPresent(
354 VOID
355 );
356
357 INIT_FUNCTION
358 VOID
359 NTAPI
360 KiSetProcessorType(VOID);
361
362 INIT_FUNCTION
363 ULONG
364 NTAPI
365 KiGetFeatureBits(VOID);
366
367 VOID
368 NTAPI
369 KiThreadStartup(VOID);
370
371 NTSTATUS
372 NTAPI
373 Ke386GetGdtEntryThread(
374 IN PKTHREAD Thread,
375 IN ULONG Offset,
376 IN PKGDTENTRY Descriptor
377 );
378
379 VOID
380 NTAPI
381 KiFlushNPXState(
382 IN FLOATING_SAVE_AREA *SaveArea
383 );
384
385 VOID
386 NTAPI
387 Ki386AdjustEsp0(
388 IN PKTRAP_FRAME TrapFrame
389 );
390
391 VOID
392 NTAPI
393 Ki386SetupAndExitToV86Mode(
394 OUT PTEB VdmTeb
395 );
396
397 INIT_FUNCTION
398 VOID
399 NTAPI
400 KeI386VdmInitialize(
401 VOID
402 );
403
404 INIT_FUNCTION
405 ULONG_PTR
406 NTAPI
407 Ki386EnableGlobalPage(
408 IN ULONG_PTR Context
409 );
410
411 INIT_FUNCTION
412 ULONG_PTR
413 NTAPI
414 Ki386EnableTargetLargePage(
415 IN ULONG_PTR Context
416 );
417
418 BOOLEAN
419 NTAPI
420 Ki386CreateIdentityMap(
421 IN PLARGE_IDENTITY_MAP IdentityMap,
422 IN PVOID StartPtr,
423 IN ULONG Length
424 );
425
426 VOID
427 NTAPI
428 Ki386FreeIdentityMap(
429 IN PLARGE_IDENTITY_MAP IdentityMap
430 );
431
432 VOID
433 NTAPI
434 Ki386EnableCurrentLargePage(
435 IN ULONG_PTR StartAddress,
436 IN ULONG Cr3
437 );
438
439 INIT_FUNCTION
440 VOID
441 NTAPI
442 KiI386PentiumLockErrataFixup(
443 VOID
444 );
445
446 INIT_FUNCTION
447 VOID
448 NTAPI
449 KiInitializePAT(
450 VOID
451 );
452
453 INIT_FUNCTION
454 VOID
455 NTAPI
456 KiInitializeMTRR(
457 IN BOOLEAN FinalCpu
458 );
459
460 INIT_FUNCTION
461 VOID
462 NTAPI
463 KiAmdK6InitializeMTRR(
464 VOID
465 );
466
467 INIT_FUNCTION
468 VOID
469 NTAPI
470 KiRestoreFastSyscallReturnState(
471 VOID
472 );
473
474 INIT_FUNCTION
475 ULONG_PTR
476 NTAPI
477 Ki386EnableDE(
478 IN ULONG_PTR Context
479 );
480
481 INIT_FUNCTION
482 ULONG_PTR
483 NTAPI
484 Ki386EnableFxsr(
485 IN ULONG_PTR Context
486 );
487
488 INIT_FUNCTION
489 ULONG_PTR
490 NTAPI
491 Ki386EnableXMMIExceptions(
492 IN ULONG_PTR Context
493 );
494
495 BOOLEAN
496 NTAPI
497 VdmDispatchBop(
498 IN PKTRAP_FRAME TrapFrame
499 );
500
501 BOOLEAN
502 NTAPI
503 VdmDispatchPageFault(
504 _In_ PKTRAP_FRAME TrapFrame
505 );
506
507 BOOLEAN
508 FASTCALL
509 KiVdmOpcodePrefix(
510 IN PKTRAP_FRAME TrapFrame,
511 IN ULONG Flags
512 );
513
514 BOOLEAN
515 FASTCALL
516 Ki386HandleOpcodeV86(
517 IN PKTRAP_FRAME TrapFrame
518 );
519
520 DECLSPEC_NORETURN
521 VOID
522 FASTCALL
523 KiEoiHelper(
524 IN PKTRAP_FRAME TrapFrame
525 );
526
527 VOID
528 FASTCALL
529 Ki386BiosCallReturnAddress(
530 IN PKTRAP_FRAME TrapFrame
531 );
532
533 ULONG_PTR
534 FASTCALL
535 KiExitV86Mode(
536 IN PKTRAP_FRAME TrapFrame
537 );
538
539 DECLSPEC_NORETURN
540 VOID
541 NTAPI
542 KiDispatchExceptionFromTrapFrame(
543 IN NTSTATUS Code,
544 IN ULONG Flags,
545 IN ULONG_PTR Address,
546 IN ULONG ParameterCount,
547 IN ULONG_PTR Parameter1,
548 IN ULONG_PTR Parameter2,
549 IN ULONG_PTR Parameter3,
550 IN PKTRAP_FRAME TrapFrame
551 );
552
553 NTSTATUS
554 NTAPI
555 KiConvertToGuiThread(
556 VOID
557 );
558
559 //
560 // Global x86 only Kernel data
561 //
562 extern PVOID Ki386IopmSaveArea;
563 extern ULONG KeI386EFlagsAndMaskV86;
564 extern ULONG KeI386EFlagsOrMaskV86;
565 extern BOOLEAN KeI386VirtualIntExtensions;
566 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
567 extern KDESCRIPTOR KiIdtDescriptor;
568 extern BOOLEAN KiI386PentiumLockErrataPresent;
569 extern ULONG KeI386NpxPresent;
570 extern ULONG KeI386XMMIPresent;
571 extern ULONG KeI386FxsrPresent;
572 extern ULONG KiMXCsrMask;
573 extern ULONG KeI386CpuType;
574 extern ULONG KeI386CpuStep;
575 extern ULONG KiFastSystemCallDisable;
576 extern UCHAR KiDebugRegisterTrapOffsets[9];
577 extern UCHAR KiDebugRegisterContextOffsets[9];
578 extern DECLSPEC_NORETURN VOID __cdecl KiTrap02(VOID);
579 extern VOID __cdecl KiTrap08(VOID);
580 extern VOID __cdecl KiTrap13(VOID);
581 extern VOID __cdecl KiFastCallEntry(VOID);
582 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
583 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
584 extern VOID __cdecl CopyParams(VOID);
585 extern VOID __cdecl ReadBatch(VOID);
586 extern CHAR KiSystemCallExitBranch[];
587 extern CHAR KiSystemCallExit[];
588 extern CHAR KiSystemCallExit2[];
589
590 //
591 // Trap Macros
592 //
593 #include "trap_x.h"
594
595 //
596 // Returns a thread's FPU save area
597 //
598 FORCEINLINE
599 PFX_SAVE_AREA
600 KiGetThreadNpxArea(IN PKTHREAD Thread)
601 {
602 ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0);
603 return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
604 }
605
606 //
607 // Sanitizes a selector
608 //
609 FORCEINLINE
610 ULONG
611 Ke386SanitizeSeg(IN ULONG Cs,
612 IN KPROCESSOR_MODE Mode)
613 {
614 //
615 // Check if we're in kernel-mode, and force CPL 0 if so.
616 // Otherwise, force CPL 3.
617 //
618 return ((Mode == KernelMode) ?
619 (Cs & (0xFFFF & ~RPL_MASK)) :
620 (RPL_MASK | (Cs & 0xFFFF)));
621 }
622
623 //
624 // Sanitizes EFLAGS
625 //
626 FORCEINLINE
627 ULONG
628 Ke386SanitizeFlags(IN ULONG Eflags,
629 IN KPROCESSOR_MODE Mode)
630 {
631 //
632 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
633 // Otherwise, also force interrupt mask on.
634 //
635 return ((Mode == KernelMode) ?
636 (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
637 (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
638 }
639
640 //
641 // Sanitizes a Debug Register
642 //
643 FORCEINLINE
644 PVOID
645 Ke386SanitizeDr(IN PVOID DrAddress,
646 IN KPROCESSOR_MODE Mode)
647 {
648 //
649 // Check if we're in kernel-mode, and return the address directly if so.
650 // Otherwise, make sure it's not inside the kernel-mode address space.
651 // If it is, then clear the address.
652 //
653 return ((Mode == KernelMode) ? DrAddress :
654 (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
655 }
656
657 //
658 // Exception with no arguments
659 //
660 FORCEINLINE
661 DECLSPEC_NORETURN
662 VOID
663 KiDispatchException0Args(IN NTSTATUS Code,
664 IN ULONG_PTR Address,
665 IN PKTRAP_FRAME TrapFrame)
666 {
667 /* Helper for exceptions with no arguments */
668 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame);
669 }
670
671 //
672 // Exception with one argument
673 //
674 FORCEINLINE
675 DECLSPEC_NORETURN
676 VOID
677 KiDispatchException1Args(IN NTSTATUS Code,
678 IN ULONG_PTR Address,
679 IN ULONG P1,
680 IN PKTRAP_FRAME TrapFrame)
681 {
682 /* Helper for exceptions with no arguments */
683 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame);
684 }
685
686 //
687 // Exception with two arguments
688 //
689 FORCEINLINE
690 DECLSPEC_NORETURN
691 VOID
692 KiDispatchException2Args(IN NTSTATUS Code,
693 IN ULONG_PTR Address,
694 IN ULONG P1,
695 IN ULONG P2,
696 IN PKTRAP_FRAME TrapFrame)
697 {
698 /* Helper for exceptions with no arguments */
699 KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame);
700 }
701
702 //
703 // Performs a system call
704 //
705
706 /*
707 * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
708 * and then calls the function associated with the system call.
709 *
710 * It's done in assembly for two reasons: we need to muck with the stack,
711 * and the call itself restores the stack back for us. The only way to do
712 * this in C is to do manual C handlers for every possible number of args on
713 * the stack, and then have the handler issue a call by pointer. This is
714 * wasteful since it'll basically push the values twice and require another
715 * level of call indirection.
716 *
717 * The ARM kernel currently does this, but it should probably be changed
718 * later to function like this as well.
719 *
720 */
721 #ifdef __GNUC__
722 FORCEINLINE
723 NTSTATUS
724 KiSystemCallTrampoline(IN PVOID Handler,
725 IN PVOID Arguments,
726 IN ULONG StackBytes)
727 {
728 NTSTATUS Result;
729
730 __asm__ __volatile__
731 (
732 "subl %1, %%esp\n\t"
733 "movl %%esp, %%edi\n\t"
734 "movl %2, %%esi\n\t"
735 "shrl $2, %1\n\t"
736 "rep movsd\n\t"
737 "call *%3\n\t"
738 "movl %%eax, %0"
739 : "=r"(Result)
740 : "c"(StackBytes),
741 "d"(Arguments),
742 "r"(Handler)
743 : "%esp", "%esi", "%edi"
744 );
745 return Result;
746 }
747 #elif defined(_MSC_VER)
748 FORCEINLINE
749 NTSTATUS
750 KiSystemCallTrampoline(IN PVOID Handler,
751 IN PVOID Arguments,
752 IN ULONG StackBytes)
753 {
754 __asm
755 {
756 mov ecx, StackBytes
757 mov esi, Arguments
758 mov eax, Handler
759 sub esp, ecx
760 mov edi, esp
761 shr ecx, 2
762 rep movsd
763 call eax
764 }
765 /* Return with result in EAX */
766 }
767 #else
768 #error Unknown Compiler
769 #endif
770
771
772 //
773 // Checks for pending APCs
774 //
775 FORCEINLINE
776 VOID
777 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
778 {
779 PKTHREAD Thread;
780 KIRQL OldIrql;
781
782 /* Check for V8086 or user-mode trap */
783 if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
784 {
785 /* Get the thread */
786 Thread = KeGetCurrentThread();
787 while (TRUE)
788 {
789 /* Turn off the alerted state for kernel mode */
790 Thread->Alerted[KernelMode] = FALSE;
791
792 /* Are there pending user APCs? */
793 if (!Thread->ApcState.UserApcPending) break;
794
795 /* Raise to APC level and enable interrupts */
796 OldIrql = KfRaiseIrql(APC_LEVEL);
797 _enable();
798
799 /* Deliver APCs */
800 KiDeliverApc(UserMode, NULL, TrapFrame);
801
802 /* Restore IRQL and disable interrupts once again */
803 KfLowerIrql(OldIrql);
804 _disable();
805 }
806 }
807 }
808
809 //
810 // Switches from boot loader to initial kernel stack
811 //
812 INIT_FUNCTION
813 FORCEINLINE
814 VOID
815 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
816 {
817 INIT_FUNCTION VOID NTAPI KiSystemStartupBootStack(VOID);
818
819 /* We have to switch to a new stack before continuing kernel initialization */
820 #ifdef __GNUC__
821 __asm__
822 (
823 "movl %0, %%esp\n\t"
824 "subl %1, %%esp\n\t"
825 "pushl %2\n\t"
826 "jmp _KiSystemStartupBootStack@0"
827 :
828 : "c"(InitialStack),
829 "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
830 "i"(CR0_EM | CR0_TS | CR0_MP),
831 "p"(KiSystemStartupBootStack)
832 : "%esp"
833 );
834 #elif defined(_MSC_VER)
835 __asm
836 {
837 mov esp, InitialStack
838 sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
839 push (CR0_EM | CR0_TS | CR0_MP)
840 jmp KiSystemStartupBootStack
841 }
842 #else
843 #error Unknown Compiler
844 #endif
845 }
846
847 //
848 // Emits the iret instruction for C code
849 //
850 FORCEINLINE
851 DECLSPEC_NORETURN
852 VOID
853 KiIret(VOID)
854 {
855 #if defined(__GNUC__)
856 __asm__ __volatile__
857 (
858 "iret"
859 );
860 #elif defined(_MSC_VER)
861 __asm
862 {
863 iretd
864 }
865 #else
866 #error Unsupported compiler
867 #endif
868 UNREACHABLE;
869 }
870
871 //
872 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
873 // initiates the end by calling back into the HAL and exiting the trap here.
874 //
875 FORCEINLINE
876 VOID
877 KiEndInterrupt(IN KIRQL Irql,
878 IN PKTRAP_FRAME TrapFrame)
879 {
880 /* Disable interrupts and end the interrupt */
881 _disable();
882 HalEndSystemInterrupt(Irql, TrapFrame);
883
884 /* Exit the interrupt */
885 KiEoiHelper(TrapFrame);
886 }
887
888 //
889 // PERF Code
890 //
891 FORCEINLINE
892 VOID
893 Ki386PerfEnd(VOID)
894 {
895 extern ULONGLONG BootCyclesEnd, BootCycles;
896 BootCyclesEnd = __rdtsc();
897 DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
898 DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
899 KeGetCurrentPrcb()->InterruptCount,
900 KeGetCurrentPrcb()->KeSystemCalls,
901 KeGetContextSwitches(KeGetCurrentPrcb()));
902 }
903
904 FORCEINLINE
905 PULONG
906 KiGetUserModeStackAddress(void)
907 {
908 return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
909 }
910
911 #endif