ntoskrnl/include/internal/i386/ke.h

   1 #pragma once
   2
   3 #ifndef __ASM__
   4
   5 #include "intrin_i.h"
   6
   7 //
   8 // Thread Dispatcher Header DebugActive Mask
   9 //
  10 #define DR_MASK(x)                              (1 << (x))
  11 #define DR_REG_MASK                             0x4F
  12
  13 //
  14 // INT3 is 1 byte long
  15 //
  16 #define KD_BREAKPOINT_TYPE        UCHAR
  17 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
  18 #define KD_BREAKPOINT_VALUE       0xCC
  19
  20 //
  21 // Macros for getting and setting special purpose registers in portable code
  22 //
  23 #define KeGetContextPc(Context) \
  24     ((Context)->Eip)
  25
  26 #define KeSetContextPc(Context, ProgramCounter) \
  27     ((Context)->Eip = (ProgramCounter))
  28
  29 #define KeGetTrapFramePc(TrapFrame) \
  30     ((TrapFrame)->Eip)
  31
  32 #define KiGetLinkedTrapFrame(x) \
  33     (PKTRAP_FRAME)((x)->Edx)
  34
  35 #define KeGetContextReturnRegister(Context) \
  36     ((Context)->Eax)
  37
  38 #define KeSetContextReturnRegister(Context, ReturnValue) \
  39     ((Context)->Eax = (ReturnValue))
  40
  41 //
  42 // Macro to get trap and exception frame from a thread stack
  43 //
  44 #define KeGetTrapFrame(Thread) \
  45     (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
  46                    sizeof(KTRAP_FRAME) - \
  47                    sizeof(FX_SAVE_AREA))
  48
  49 #define KeGetExceptionFrame(Thread) \
  50     NULL
  51
  52 //
  53 // Macro to get context switches from the PRCB
  54 // All architectures but x86 have it in the PRCB's KeContextSwitches
  55 //
  56 #define KeGetContextSwitches(Prcb)  \
  57     CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
  58
  59 //
  60 // Macro to get the second level cache size field name which differs between
  61 // CISC and RISC architectures, as the former has unified I/D cache
  62 //
  63 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
  64
  65 //
  66 // Returns the Interrupt State from a Trap Frame.
  67 // ON = TRUE, OFF = FALSE
  68 //
  69 #define KeGetTrapFrameInterruptState(TrapFrame) \
  70         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
  71
  72 //
  73 // Flags for exiting a trap
  74 //
  75 #define KTE_SKIP_PM_BIT  (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
  76 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments  = TRUE } }).Bits)
  77 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
  78
  79 typedef union _KTRAP_EXIT_SKIP_BITS
  80 {
  81     struct
  82     {
  83         UCHAR SkipPreviousMode:1;
  84         UCHAR SkipSegments:1;
  85         UCHAR SkipVolatiles:1;
  86         UCHAR Reserved:5;
  87     };
  88     UCHAR Bits;
  89 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
  90
  91
  92 //
  93 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
  94 //
  95 #define PFX_FLAG_ES                0x00000100
  96 #define PFX_FLAG_CS                0x00000200
  97 #define PFX_FLAG_SS                0x00000400
  98 #define PFX_FLAG_DS                0x00000800
  99 #define PFX_FLAG_FS                0x00001000
 100 #define PFX_FLAG_GS                0x00002000
 101 #define PFX_FLAG_OPER32            0x00004000
 102 #define PFX_FLAG_ADDR32            0x00008000
 103 #define PFX_FLAG_LOCK              0x00010000
 104 #define PFX_FLAG_REPNE             0x00020000
 105 #define PFX_FLAG_REP               0x00040000
 106
 107 //
 108 // VDM Helper Macros
 109 //
 110 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
 111 // We need to keep 2 parameters while the original ASM implementation uses 4:
 112 // TrapFrame, PrefixFlags, Eip, InstructionSize;
 113 //
 114 // We pass the trap frame, and prefix flags, in our two parameters.
 115 //
 116 // We then realize that since the smallest prefix flag is 0x100, this gives us
 117 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
 118 //
 119 // We further realize that we always have access to EIP from the trap frame, and
 120 // that if we want the *current instruction* EIP, we simply have to add the
 121 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
 122 // at now, so we don't need to use the stack to push this parameter.
 123 //
 124 // We actually only care about the *current instruction* EIP in one location,
 125 // so although it may be slightly more expensive to re-calculate the EIP one
 126 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
 127 // which would be forcing stack usage in all other scenarios.
 128 //
 129 #define KiVdmSetVdmEFlags(x)        InterlockedOr((PLONG)KiNtVdmState, (x));
 130 #define KiVdmClearVdmEFlags(x)      InterlockedAnd((PLONG)KiNtVdmState, ~(x))
 131 #define KiCallVdmHandler(x)         KiVdmOpcode##x(TrapFrame, Flags)
 132 #define KiCallVdmPrefixHandler(x)   KiVdmOpcodePrefix(TrapFrame, Flags | x)
 133 #define KiVdmUnhandledOpcode(x)                     \
 134     BOOLEAN                                         \
 135     FASTCALL                                        \
 136     KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame,       \
 137                    IN ULONG Flags)                  \
 138     {                                               \
 139         /* Not yet handled */                       \
 140         UNIMPLEMENTED_DBGBREAK();                   \
 141         return FALSE;                               \
 142     }
 143
 144 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
 145
 146 //
 147 // Local parameters
 148 //
 149 typedef struct _KV86_FRAME
 150 {
 151     PVOID ThreadStack;
 152     PVOID ThreadTeb;
 153     PVOID PcrTeb;
 154 } KV86_FRAME, *PKV86_FRAME;
 155
 156 //
 157 // Virtual Stack Frame
 158 //
 159 typedef struct _KV8086_STACK_FRAME
 160 {
 161     KTRAP_FRAME TrapFrame;
 162     FX_SAVE_AREA NpxArea;
 163     KV86_FRAME V86Frame;
 164 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
 165
 166 //
 167 // Large Pages Support
 168 //
 169 typedef struct _LARGE_IDENTITY_MAP
 170 {
 171     PHARDWARE_PTE TopLevelDirectory;
 172     ULONG Cr3;
 173     ULONG_PTR StartAddress;
 174     ULONG PagesCount;
 175     PVOID PagesList[30];
 176 } LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP;
 177
 178 /* Diable interrupts and return whether they were enabled before */
 179 FORCEINLINE
 180 BOOLEAN
 181 KeDisableInterrupts(VOID)
 182 {
 183     ULONG Flags;
 184     BOOLEAN Return;
 185
 186     /* Get EFLAGS and check if the interrupt bit is set */
 187     Flags = __readeflags();
 188     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 189
 190     /* Disable interrupts */
 191     _disable();
 192     return Return;
 193 }
 194
 195 /* Restore previous interrupt state */
 196 FORCEINLINE
 197 VOID
 198 KeRestoreInterrupts(BOOLEAN WereEnabled)
 199 {
 200     if (WereEnabled) _enable();
 201 }
 202
 203 //
 204 // Registers an interrupt handler with an IDT vector
 205 //
 206 FORCEINLINE
 207 VOID
 208 KeRegisterInterruptHandler(IN ULONG Vector,
 209                            IN PVOID Handler)
 210 {
 211     UCHAR Entry;
 212     ULONG_PTR Address;
 213     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 214
 215     //
 216     // Get the entry from the HAL
 217     //
 218     Entry = HalVectorToIDTEntry(Vector);
 219     Address = PtrToUlong(Handler);
 220
 221     //
 222     // Now set the data
 223     //
 224     Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
 225     Pcr->IDT[Entry].Offset = (USHORT)Address;
 226 }
 227
 228 //
 229 // Returns the registered interrupt handler for a given IDT vector
 230 //
 231 FORCEINLINE
 232 PVOID
 233 KeQueryInterruptHandler(IN ULONG Vector)
 234 {
 235     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 236     UCHAR Entry;
 237
 238     //
 239     // Get the entry from the HAL
 240     //
 241     Entry = HalVectorToIDTEntry(Vector);
 242
 243     //
 244     // Read the entry from the IDT
 245     //
 246     return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
 247                     (Pcr->IDT[Entry].Offset & 0xFFFF));
 248 }
 249
 250 //
 251 // Invalidates the TLB entry for a specified address
 252 //
 253 FORCEINLINE
 254 VOID
 255 KeInvalidateTlbEntry(IN PVOID Address)
 256 {
 257     /* Invalidate the TLB entry for this address */
 258     __invlpg(Address);
 259 }
 260
 261 FORCEINLINE
 262 VOID
 263 KeFlushProcessTb(VOID)
 264 {
 265     /* Flush the TLB by resetting CR3 */
 266     __writecr3(__readcr3());
 267 }
 268
 269 FORCEINLINE
 270 PRKTHREAD
 271 KeGetCurrentThread(VOID)
 272 {
 273     /* Return the current thread */
 274     return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
 275 }
 276
 277 FORCEINLINE
 278 VOID
 279 KiRundownThread(IN PKTHREAD Thread)
 280 {
 281 #ifndef CONFIG_SMP
 282     /* Check if this is the NPX Thread */
 283     if (KeGetCurrentPrcb()->NpxThread == Thread)
 284     {
 285         /* Clear it */
 286         KeGetCurrentPrcb()->NpxThread = NULL;
 287         Ke386FnInit();
 288     }
 289 #else
 290     /* Nothing to do */
 291 #endif
 292 }
 293
 294 FORCEINLINE
 295 VOID
 296 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)
 297 {
 298     GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF);
 299     GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16);
 300     GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24);
 301 }
 302
 303 FORCEINLINE
 304 VOID
 305 KiSetTebBase(PKPCR Pcr, PVOID TebAddress)
 306 {
 307     Pcr->NtTib.Self = TebAddress;
 308     Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress);
 309 }
 310
 311 VOID
 312 FASTCALL
 313 Ki386InitializeTss(
 314     IN PKTSS Tss,
 315     IN PKIDTENTRY Idt,
 316     IN PKGDTENTRY Gdt
 317 );
 318
 319 VOID
 320 NTAPI
 321 KiSetCR0Bits(VOID);
 322
 323 VOID
 324 NTAPI
 325 KiGetCacheInformation(VOID);
 326
 327 BOOLEAN
 328 NTAPI
 329 KiIsNpxPresent(
 330     VOID
 331 );
 332
 333 BOOLEAN
 334 NTAPI
 335 KiIsNpxErrataPresent(
 336     VOID
 337 );
 338
 339 VOID
 340 NTAPI
 341 KiSetProcessorType(VOID);
 342
 343 ULONG
 344 NTAPI
 345 KiGetFeatureBits(VOID);
 346
 347 VOID
 348 NTAPI
 349 KiThreadStartup(VOID);
 350
 351 NTSTATUS
 352 NTAPI
 353 Ke386GetGdtEntryThread(
 354     IN PKTHREAD Thread,
 355     IN ULONG Offset,
 356     IN PKGDTENTRY Descriptor
 357 );
 358
 359 VOID
 360 NTAPI
 361 KiFlushNPXState(
 362     IN FLOATING_SAVE_AREA *SaveArea
 363 );
 364
 365 VOID
 366 NTAPI
 367 Ki386AdjustEsp0(
 368     IN PKTRAP_FRAME TrapFrame
 369 );
 370
 371 VOID
 372 NTAPI
 373 Ki386SetupAndExitToV86Mode(
 374     OUT PTEB VdmTeb
 375 );
 376
 377 VOID
 378 NTAPI
 379 KeI386VdmInitialize(
 380     VOID
 381 );
 382
 383 ULONG_PTR
 384 NTAPI
 385 Ki386EnableGlobalPage(
 386     IN ULONG_PTR Context
 387 );
 388
 389 ULONG_PTR
 390 NTAPI
 391 Ki386EnableTargetLargePage(
 392     IN ULONG_PTR Context
 393 );
 394
 395 BOOLEAN
 396 NTAPI
 397 Ki386CreateIdentityMap(
 398     IN PLARGE_IDENTITY_MAP IdentityMap,
 399     IN PVOID StartPtr,
 400     IN ULONG Length
 401 );
 402
 403 VOID
 404 NTAPI
 405 Ki386FreeIdentityMap(
 406     IN PLARGE_IDENTITY_MAP IdentityMap
 407 );
 408
 409 VOID
 410 NTAPI
 411 Ki386EnableCurrentLargePage(
 412     IN ULONG_PTR StartAddress,
 413     IN ULONG Cr3
 414 );
 415
 416 VOID
 417 NTAPI
 418 KiI386PentiumLockErrataFixup(
 419     VOID
 420 );
 421
 422 VOID
 423 NTAPI
 424 KiInitializePAT(
 425     VOID
 426 );
 427
 428 VOID
 429 NTAPI
 430 KiInitializeMTRR(
 431     IN BOOLEAN FinalCpu
 432 );
 433
 434 VOID
 435 NTAPI
 436 KiAmdK6InitializeMTRR(
 437     VOID
 438 );
 439
 440 VOID
 441 NTAPI
 442 KiRestoreFastSyscallReturnState(
 443     VOID
 444 );
 445
 446 ULONG_PTR
 447 NTAPI
 448 Ki386EnableDE(
 449     IN ULONG_PTR Context
 450 );
 451
 452 ULONG_PTR
 453 NTAPI
 454 Ki386EnableFxsr(
 455     IN ULONG_PTR Context
 456 );
 457
 458 ULONG_PTR
 459 NTAPI
 460 Ki386EnableXMMIExceptions(
 461     IN ULONG_PTR Context
 462 );
 463
 464 BOOLEAN
 465 NTAPI
 466 VdmDispatchBop(
 467     IN PKTRAP_FRAME TrapFrame
 468 );
 469
 470 BOOLEAN
 471 FASTCALL
 472 KiVdmOpcodePrefix(
 473     IN PKTRAP_FRAME TrapFrame,
 474     IN ULONG Flags
 475 );
 476
 477 BOOLEAN
 478 FASTCALL
 479 Ki386HandleOpcodeV86(
 480     IN PKTRAP_FRAME TrapFrame
 481 );
 482
 483 DECLSPEC_NORETURN
 484 VOID
 485 FASTCALL
 486 KiEoiHelper(
 487     IN PKTRAP_FRAME TrapFrame
 488 );
 489
 490 VOID
 491 FASTCALL
 492 Ki386BiosCallReturnAddress(
 493     IN PKTRAP_FRAME TrapFrame
 494 );
 495
 496 ULONG_PTR
 497 FASTCALL
 498 KiExitV86Mode(
 499     IN PKTRAP_FRAME TrapFrame
 500 );
 501
 502 DECLSPEC_NORETURN
 503 VOID
 504 NTAPI
 505 KiDispatchExceptionFromTrapFrame(
 506     IN NTSTATUS Code,
 507     IN ULONG Flags,
 508     IN ULONG_PTR Address,
 509     IN ULONG ParameterCount,
 510     IN ULONG_PTR Parameter1,
 511     IN ULONG_PTR Parameter2,
 512     IN ULONG_PTR Parameter3,
 513     IN PKTRAP_FRAME TrapFrame
 514 );
 515
 516 //
 517 // Global x86 only Kernel data
 518 //
 519 extern PVOID Ki386IopmSaveArea;
 520 extern ULONG KeI386EFlagsAndMaskV86;
 521 extern ULONG KeI386EFlagsOrMaskV86;
 522 extern BOOLEAN KeI386VirtualIntExtensions;
 523 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
 524 extern KDESCRIPTOR KiIdtDescriptor;
 525 extern BOOLEAN KiI386PentiumLockErrataPresent;
 526 extern ULONG KeI386NpxPresent;
 527 extern ULONG KeI386XMMIPresent;
 528 extern ULONG KeI386FxsrPresent;
 529 extern ULONG KiMXCsrMask;
 530 extern ULONG KeI386CpuType;
 531 extern ULONG KeI386CpuStep;
 532 extern ULONG Ke386CacheAlignment;
 533 extern ULONG KiFastSystemCallDisable;
 534 extern UCHAR KiDebugRegisterTrapOffsets[9];
 535 extern UCHAR KiDebugRegisterContextOffsets[9];
 536 extern DECLSPEC_NORETURN VOID __cdecl KiTrap02(VOID);
 537 extern VOID __cdecl KiTrap08(VOID);
 538 extern VOID __cdecl KiTrap13(VOID);
 539 extern VOID __cdecl KiFastCallEntry(VOID);
 540 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
 541 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
 542 extern VOID __cdecl CopyParams(VOID);
 543 extern VOID __cdecl ReadBatch(VOID);
 544 extern VOID __cdecl FrRestore(VOID);
 545 extern CHAR KiSystemCallExitBranch[];
 546 extern CHAR KiSystemCallExit[];
 547 extern CHAR KiSystemCallExit2[];
 548
 549 //
 550 // Trap Macros
 551 //
 552 #include "trap_x.h"
 553
 554 //
 555 // Returns a thread's FPU save area
 556 //
 557 FORCEINLINE
 558 PFX_SAVE_AREA
 559 KiGetThreadNpxArea(IN PKTHREAD Thread)
 560 {
 561     ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0);
 562     return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
 563 }
 564
 565 //
 566 // Sanitizes a selector
 567 //
 568 FORCEINLINE
 569 ULONG
 570 Ke386SanitizeSeg(IN ULONG Cs,
 571                 IN KPROCESSOR_MODE Mode)
 572 {
 573     //
 574     // Check if we're in kernel-mode, and force CPL 0 if so.
 575     // Otherwise, force CPL 3.
 576     //
 577     return ((Mode == KernelMode) ?
 578             (Cs & (0xFFFF & ~RPL_MASK)) :
 579             (RPL_MASK | (Cs & 0xFFFF)));
 580 }
 581
 582 //
 583 // Sanitizes EFLAGS
 584 //
 585 FORCEINLINE
 586 ULONG
 587 Ke386SanitizeFlags(IN ULONG Eflags,
 588                    IN KPROCESSOR_MODE Mode)
 589 {
 590     //
 591     // Check if we're in kernel-mode, and sanitize EFLAGS if so.
 592     // Otherwise, also force interrupt mask on.
 593     //
 594     return ((Mode == KernelMode) ?
 595             (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
 596             (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
 597 }
 598
 599 //
 600 // Sanitizes a Debug Register
 601 //
 602 FORCEINLINE
 603 PVOID
 604 Ke386SanitizeDr(IN PVOID DrAddress,
 605                 IN KPROCESSOR_MODE Mode)
 606 {
 607     //
 608     // Check if we're in kernel-mode, and return the address directly if so.
 609     // Otherwise, make sure it's not inside the kernel-mode address space.
 610     // If it is, then clear the address.
 611     //
 612     return ((Mode == KernelMode) ? DrAddress :
 613             (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
 614 }
 615
 616 //
 617 // Exception with no arguments
 618 //
 619 FORCEINLINE
 620 DECLSPEC_NORETURN
 621 VOID
 622 KiDispatchException0Args(IN NTSTATUS Code,
 623                          IN ULONG_PTR Address,
 624                          IN PKTRAP_FRAME TrapFrame)
 625 {
 626     /* Helper for exceptions with no arguments */
 627     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame);
 628 }
 629
 630 //
 631 // Exception with one argument
 632 //
 633 FORCEINLINE
 634 DECLSPEC_NORETURN
 635 VOID
 636 KiDispatchException1Args(IN NTSTATUS Code,
 637                          IN ULONG_PTR Address,
 638                          IN ULONG P1,
 639                          IN PKTRAP_FRAME TrapFrame)
 640 {
 641     /* Helper for exceptions with no arguments */
 642     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame);
 643 }
 644
 645 //
 646 // Exception with two arguments
 647 //
 648 FORCEINLINE
 649 DECLSPEC_NORETURN
 650 VOID
 651 KiDispatchException2Args(IN NTSTATUS Code,
 652                          IN ULONG_PTR Address,
 653                          IN ULONG P1,
 654                          IN ULONG P2,
 655                          IN PKTRAP_FRAME TrapFrame)
 656 {
 657     /* Helper for exceptions with no arguments */
 658     KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame);
 659 }
 660
 661 //
 662 // Performs a system call
 663 //
 664
 665     /*
 666      * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
 667      * and then calls the function associated with the system call.
 668      *
 669      * It's done in assembly for two reasons: we need to muck with the stack,
 670      * and the call itself restores the stack back for us. The only way to do
 671      * this in C is to do manual C handlers for every possible number of args on
 672      * the stack, and then have the handler issue a call by pointer. This is
 673      * wasteful since it'll basically push the values twice and require another
 674      * level of call indirection.
 675      *
 676      * The ARM kernel currently does this, but it should probably be changed
 677      * later to function like this as well.
 678      *
 679      */
 680 #ifdef __GNUC__
 681 FORCEINLINE
 682 NTSTATUS
 683 KiSystemCallTrampoline(IN PVOID Handler,
 684                        IN PVOID Arguments,
 685                        IN ULONG StackBytes)
 686 {
 687     NTSTATUS Result;
 688
 689     __asm__ __volatile__
 690     (
 691         "subl %1, %%esp\n\t"
 692         "movl %%esp, %%edi\n\t"
 693         "movl %2, %%esi\n\t"
 694         "shrl $2, %1\n\t"
 695         "rep movsd\n\t"
 696         "call *%3\n\t"
 697         "movl %%eax, %0"
 698         : "=r"(Result)
 699         : "c"(StackBytes),
 700           "d"(Arguments),
 701           "r"(Handler)
 702         : "%esp", "%esi", "%edi"
 703     );
 704     return Result;
 705 }
 706 #elif defined(_MSC_VER)
 707 FORCEINLINE
 708 NTSTATUS
 709 KiSystemCallTrampoline(IN PVOID Handler,
 710                        IN PVOID Arguments,
 711                        IN ULONG StackBytes)
 712 {
 713     __asm
 714     {
 715         mov ecx, StackBytes
 716         mov esi, Arguments
 717         mov eax, Handler
 718         sub esp, ecx
 719         mov edi, esp
 720         shr ecx, 2
 721         rep movsd
 722         call eax
 723     }
 724     /* Return with result in EAX */
 725 }
 726 #else
 727 #error Unknown Compiler
 728 #endif
 729
 730
 731 //
 732 // Checks for pending APCs
 733 //
 734 FORCEINLINE
 735 VOID
 736 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
 737 {
 738     PKTHREAD Thread;
 739     KIRQL OldIrql;
 740
 741     /* Check for V8086 or user-mode trap */
 742     if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
 743     {
 744         /* Get the thread */
 745         Thread = KeGetCurrentThread();
 746         while (TRUE)
 747         {
 748             /* Turn off the alerted state for kernel mode */
 749             Thread->Alerted[KernelMode] = FALSE;
 750
 751             /* Are there pending user APCs? */
 752             if (!Thread->ApcState.UserApcPending) break;
 753
 754             /* Raise to APC level and enable interrupts */
 755             OldIrql = KfRaiseIrql(APC_LEVEL);
 756             _enable();
 757
 758             /* Deliver APCs */
 759             KiDeliverApc(UserMode, NULL, TrapFrame);
 760
 761             /* Restore IRQL and disable interrupts once again */
 762             KfLowerIrql(OldIrql);
 763             _disable();
 764         }
 765     }
 766 }
 767
 768 //
 769 // Converts a base thread to a GUI thread
 770 //
 771 #ifdef __GNUC__
 772 FORCEINLINE
 773 NTSTATUS
 774 KiConvertToGuiThread(VOID)
 775 {
 776     NTSTATUS NTAPI PsConvertToGuiThread(VOID);
 777     NTSTATUS Result;
 778     PVOID StackFrame;
 779
 780     /*
 781      * Converting to a GUI thread safely updates ESP in-place as well as the
 782      * current Thread->TrapFrame and EBP when KeSwitchKernelStack is called.
 783      *
 784      * However, PsConvertToGuiThread "helpfully" restores EBP to the original
 785      * caller's value, since it is considered a nonvolatile register. As such,
 786      * as soon as we're back after the conversion and we try to store the result
 787      * which will probably be in some stack variable (EBP-based), we'll crash as
 788      * we are touching the de-allocated non-expanded stack.
 789      *
 790      * Thus we need a way to update our EBP before EBP is touched, and the only
 791      * way to guarantee this is to do the call itself in assembly, use the EAX
 792      * register to store the result, fixup EBP, and then let the C code continue
 793      * on its merry way.
 794      *
 795      */
 796     __asm__ __volatile__
 797     (
 798         "movl %%ebp, %1\n\t"
 799         "subl %%esp, %1\n\t"
 800         "call _PsConvertToGuiThread@0\n\t"
 801         "addl %%esp, %1\n\t"
 802         "movl %1, %%ebp"
 803         : "=a"(Result), "=r"(StackFrame)
 804         : "p"(PsConvertToGuiThread)
 805         : "%esp", "%ecx", "%edx", "memory"
 806     );
 807     return Result;
 808 }
 809 #elif defined(_MSC_VER)
 810 NTSTATUS
 811 NTAPI
 812 KiConvertToGuiThread(VOID);
 813 #else
 814 #error Unknown Compiler
 815 #endif
 816
 817 //
 818 // Switches from boot loader to initial kernel stack
 819 //
 820 FORCEINLINE
 821 VOID
 822 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
 823 {
 824     VOID NTAPI KiSystemStartupBootStack(VOID);
 825
 826     /* We have to switch to a new stack before continuing kernel initialization */
 827 #ifdef __GNUC__
 828     __asm__
 829     (
 830         "movl %0, %%esp\n\t"
 831         "subl %1, %%esp\n\t"
 832         "pushl %2\n\t"
 833         "jmp _KiSystemStartupBootStack@0"
 834         :
 835         : "c"(InitialStack),
 836           "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
 837           "i"(CR0_EM | CR0_TS | CR0_MP),
 838           "p"(KiSystemStartupBootStack)
 839         : "%esp"
 840     );
 841 #elif defined(_MSC_VER)
 842     __asm
 843     {
 844         mov esp, InitialStack
 845         sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
 846         push (CR0_EM | CR0_TS | CR0_MP)
 847         jmp KiSystemStartupBootStack
 848     }
 849 #else
 850 #error Unknown Compiler
 851 #endif
 852 }
 853
 854 //
 855 // Emits the iret instruction for C code
 856 //
 857 FORCEINLINE
 858 DECLSPEC_NORETURN
 859 VOID
 860 KiIret(VOID)
 861 {
 862 #if defined(__GNUC__)
 863     __asm__ __volatile__
 864     (
 865         "iret"
 866     );
 867 #elif defined(_MSC_VER)
 868     __asm
 869     {
 870         iretd
 871     }
 872 #else
 873 #error Unsupported compiler
 874 #endif
 875     UNREACHABLE;
 876 }
 877
 878 //
 879 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
 880 // initiates the end by calling back into the HAL and exiting the trap here.
 881 //
 882 FORCEINLINE
 883 VOID
 884 KiEndInterrupt(IN KIRQL Irql,
 885                IN PKTRAP_FRAME TrapFrame)
 886 {
 887     /* Disable interrupts and end the interrupt */
 888     _disable();
 889     HalEndSystemInterrupt(Irql, TrapFrame);
 890
 891     /* Exit the interrupt */
 892     KiEoiHelper(TrapFrame);
 893 }
 894
 895 //
 896 // PERF Code
 897 //
 898 FORCEINLINE
 899 VOID
 900 Ki386PerfEnd(VOID)
 901 {
 902     extern ULONGLONG BootCyclesEnd, BootCycles;
 903     BootCyclesEnd = __rdtsc();
 904     DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles);
 905     DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n",
 906              KeGetCurrentPrcb()->InterruptCount,
 907              KeGetCurrentPrcb()->KeSystemCalls,
 908              KeGetContextSwitches(KeGetCurrentPrcb()));
 909 }
 910
 911 FORCEINLINE
 912 PULONG
 913 KiGetUserModeStackAddress(void)
 914 {
 915     return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
 916 }
 917
 918 #endif