reactos/ntoskrnl/include/internal/i386/ke.h

   1 #pragma once
   2
   3 #ifndef __ASM__
   4
   5 #include "intrin_i.h"
   6
   7 //
   8 // Thread Dispatcher Header DebugActive Mask
   9 //
  10 #define DR_MASK(x)                              (1 << (x))
  11 #define DR_REG_MASK                             0x4F
  12
  13 //
  14 // INT3 is 1 byte long
  15 //
  16 #define KD_BREAKPOINT_TYPE        UCHAR
  17 #define KD_BREAKPOINT_SIZE        sizeof(UCHAR)
  18 #define KD_BREAKPOINT_VALUE       0xCC
  19
  20 //
  21 // Macros for getting and setting special purpose registers in portable code
  22 //
  23 #define KeGetContextPc(Context) \
  24     ((Context)->Eip)
  25
  26 #define KeSetContextPc(Context, ProgramCounter) \
  27     ((Context)->Eip = (ProgramCounter))
  28
  29 #define KeGetTrapFramePc(TrapFrame) \
  30     ((TrapFrame)->Eip)
  31
  32 #define KiGetLinkedTrapFrame(x) \
  33     (PKTRAP_FRAME)((x)->Edx)
  34
  35 #define KeGetContextReturnRegister(Context) \
  36     ((Context)->Eax)
  37
  38 #define KeSetContextReturnRegister(Context, ReturnValue) \
  39     ((Context)->Eax = (ReturnValue))
  40
  41 //
  42 // Macro to get trap and exception frame from a thread stack
  43 //
  44 #define KeGetTrapFrame(Thread) \
  45     (PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
  46                    sizeof(KTRAP_FRAME) - \
  47                    sizeof(FX_SAVE_AREA))
  48
  49 #define KeGetExceptionFrame(Thread) \
  50     NULL
  51
  52 //
  53 // Macro to get context switches from the PRCB
  54 // All architectures but x86 have it in the PRCB's KeContextSwitches
  55 //
  56 #define KeGetContextSwitches(Prcb)  \
  57     CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
  58
  59 //
  60 // Macro to get the second level cache size field name which differs between
  61 // CISC and RISC architectures, as the former has unified I/D cache
  62 //
  63 #define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
  64
  65 //
  66 // Returns the Interrupt State from a Trap Frame.
  67 // ON = TRUE, OFF = FALSE
  68 //
  69 #define KeGetTrapFrameInterruptState(TrapFrame) \
  70         BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
  71
  72 //
  73 // Flags for exiting a trap
  74 //
  75 #define KTE_SKIP_PM_BIT  (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
  76 #define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments  = TRUE } }).Bits)
  77 #define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
  78
  79 typedef union _KTRAP_EXIT_SKIP_BITS
  80 {
  81     struct
  82     {
  83         UCHAR SkipPreviousMode:1;
  84         UCHAR SkipSegments:1;
  85         UCHAR SkipVolatiles:1;
  86         UCHAR Reserved:5;
  87     };
  88     UCHAR Bits;
  89 } KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
  90
  91
  92 //
  93 // Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
  94 //
  95 #define PFX_FLAG_ES                0x00000100
  96 #define PFX_FLAG_CS                0x00000200
  97 #define PFX_FLAG_SS                0x00000400
  98 #define PFX_FLAG_DS                0x00000800
  99 #define PFX_FLAG_FS                0x00001000
 100 #define PFX_FLAG_GS                0x00002000
 101 #define PFX_FLAG_OPER32            0x00004000
 102 #define PFX_FLAG_ADDR32            0x00008000
 103 #define PFX_FLAG_LOCK              0x00010000
 104 #define PFX_FLAG_REPNE             0x00020000
 105 #define PFX_FLAG_REP               0x00040000
 106
 107 //
 108 // VDM Helper Macros
 109 //
 110 // All VDM/V8086 opcode emulators have the same FASTCALL function definition.
 111 // We need to keep 2 parameters while the original ASM implementation uses 4:
 112 // TrapFrame, PrefixFlags, Eip, InstructionSize;
 113 //
 114 // We pass the trap frame, and prefix flags, in our two parameters.
 115 //
 116 // We then realize that since the smallest prefix flag is 0x100, this gives us
 117 // a count of up to 0xFF. So we OR in the instruction size with the prefix flags
 118 //
 119 // We further realize that we always have access to EIP from the trap frame, and
 120 // that if we want the *current instruction* EIP, we simply have to add the
 121 // instruction size *MINUS ONE*, and that gives us the EIP we should be looking
 122 // at now, so we don't need to use the stack to push this parameter.
 123 //
 124 // We actually only care about the *current instruction* EIP in one location,
 125 // so although it may be slightly more expensive to re-calculate the EIP one
 126 // more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
 127 // which would be forcing stack usage in all other scenarios.
 128 //
 129 #define KiVdmSetVdmEFlags(x)        InterlockedOr((PLONG)KiNtVdmState, (x));
 130 #define KiVdmClearVdmEFlags(x)      InterlockedAnd((PLONG)KiNtVdmState, ~(x))
 131 #define KiCallVdmHandler(x)         KiVdmOpcode##x(TrapFrame, Flags)
 132 #define KiCallVdmPrefixHandler(x)   KiVdmOpcodePrefix(TrapFrame, Flags | x)
 133 #define KiVdmUnhandledOpcode(x)                     \
 134     BOOLEAN                                         \
 135     FASTCALL                                        \
 136     KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame,       \
 137                    IN ULONG Flags)                  \
 138     {                                               \
 139         /* Not yet handled */                       \
 140         UNIMPLEMENTED;                              \
 141         while (TRUE);                               \
 142         return TRUE;                                \
 143     }
 144
 145 C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
 146
 147 //
 148 // Local parameters
 149 //
 150 typedef struct _KV86_FRAME
 151 {
 152     PVOID ThreadStack;
 153     PVOID ThreadTeb;
 154     PVOID PcrTeb;
 155 } KV86_FRAME, *PKV86_FRAME;
 156
 157 //
 158 // Virtual Stack Frame
 159 //
 160 typedef struct _KV8086_STACK_FRAME
 161 {
 162     KTRAP_FRAME TrapFrame;
 163     FX_SAVE_AREA NpxArea;
 164     KV86_FRAME V86Frame;
 165 } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
 166
 167 /* Diable interrupts and return whether they were enabled before */
 168 FORCEINLINE
 169 BOOLEAN
 170 KeDisableInterrupts(VOID)
 171 {
 172     ULONG Flags;
 173     BOOLEAN Return;
 174
 175     /* Get EFLAGS and check if the interrupt bit is set */
 176     Flags = __readeflags();
 177     Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
 178
 179     /* Disable interrupts */
 180     _disable();
 181     return Return;
 182 }
 183
 184 /* Restore previous interrupt state */
 185 FORCEINLINE
 186 VOID
 187 KeRestoreInterrupts(BOOLEAN WereEnabled)
 188 {
 189     if (WereEnabled) _enable();
 190 }
 191
 192 //
 193 // Registers an interrupt handler with an IDT vector
 194 //
 195 FORCEINLINE
 196 VOID
 197 KeRegisterInterruptHandler(IN ULONG Vector,
 198                            IN PVOID Handler)
 199 {
 200     UCHAR Entry;
 201     ULONG_PTR Address;
 202     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 203
 204     //
 205     // Get the entry from the HAL
 206     //
 207     Entry = HalVectorToIDTEntry(Vector);
 208     Address = PtrToUlong(Handler);
 209
 210     //
 211     // Now set the data
 212     //
 213     Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
 214     Pcr->IDT[Entry].Offset = (USHORT)Address;
 215 }
 216
 217 //
 218 // Returns the registered interrupt handler for a given IDT vector
 219 //
 220 FORCEINLINE
 221 PVOID
 222 KeQueryInterruptHandler(IN ULONG Vector)
 223 {
 224     PKIPCR Pcr = (PKIPCR)KeGetPcr();
 225     UCHAR Entry;
 226
 227     //
 228     // Get the entry from the HAL
 229     //
 230     Entry = HalVectorToIDTEntry(Vector);
 231
 232     //
 233     // Read the entry from the IDT
 234     //
 235     return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
 236                     (Pcr->IDT[Entry].Offset & 0xFFFF));
 237 }
 238
 239 //
 240 // Invalidates the TLB entry for a specified address
 241 //
 242 FORCEINLINE
 243 VOID
 244 KeInvalidateTlbEntry(IN PVOID Address)
 245 {
 246     /* Invalidate the TLB entry for this address */
 247     __invlpg(Address);
 248 }
 249
 250 FORCEINLINE
 251 VOID
 252 KeFlushProcessTb(VOID)
 253 {
 254     /* Flush the TLB by resetting CR3 */
 255     __writecr3(__readcr3());
 256 }
 257
 258 FORCEINLINE
 259 PRKTHREAD
 260 KeGetCurrentThread(VOID)
 261 {
 262     /* Return the current thread */
 263     return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
 264 }
 265
 266 FORCEINLINE
 267 VOID
 268 KiRundownThread(IN PKTHREAD Thread)
 269 {
 270 #ifndef CONFIG_SMP
 271     /* Check if this is the NPX Thread */
 272     if (KeGetCurrentPrcb()->NpxThread == Thread)
 273     {
 274         /* Clear it */
 275         KeGetCurrentPrcb()->NpxThread = NULL;
 276         Ke386FnInit();
 277     }
 278 #else
 279     /* Nothing to do */
 280 #endif
 281 }
 282
 283 VOID
 284 FASTCALL
 285 Ki386InitializeTss(
 286     IN PKTSS Tss,
 287     IN PKIDTENTRY Idt,
 288     IN PKGDTENTRY Gdt
 289 );
 290
 291 VOID
 292 NTAPI
 293 KiSetCR0Bits(VOID);
 294
 295 VOID
 296 NTAPI
 297 KiGetCacheInformation(VOID);
 298
 299 BOOLEAN
 300 NTAPI
 301 KiIsNpxPresent(
 302     VOID
 303 );
 304
 305 BOOLEAN
 306 NTAPI
 307 KiIsNpxErrataPresent(
 308     VOID
 309 );
 310
 311 VOID
 312 NTAPI
 313 KiSetProcessorType(VOID);
 314
 315 ULONG
 316 NTAPI
 317 KiGetFeatureBits(VOID);
 318
 319 VOID
 320 NTAPI
 321 KiThreadStartup(VOID);
 322
 323 NTSTATUS
 324 NTAPI
 325 Ke386GetGdtEntryThread(
 326     IN PKTHREAD Thread,
 327     IN ULONG Offset,
 328     IN PKGDTENTRY Descriptor
 329 );
 330
 331 VOID
 332 NTAPI
 333 KiFlushNPXState(
 334     IN FLOATING_SAVE_AREA *SaveArea
 335 );
 336
 337 VOID
 338 NTAPI
 339 Ki386AdjustEsp0(
 340     IN PKTRAP_FRAME TrapFrame
 341 );
 342
 343 VOID
 344 NTAPI
 345 Ki386SetupAndExitToV86Mode(
 346     OUT PTEB VdmTeb
 347 );
 348
 349 VOID
 350 NTAPI
 351 KeI386VdmInitialize(
 352     VOID
 353 );
 354
 355 ULONG_PTR
 356 NTAPI
 357 Ki386EnableGlobalPage(
 358     IN volatile ULONG_PTR Context
 359 );
 360
 361 VOID
 362 NTAPI
 363 KiI386PentiumLockErrataFixup(
 364     VOID
 365 );
 366
 367 VOID
 368 NTAPI
 369 KiInitializePAT(
 370     VOID
 371 );
 372
 373 VOID
 374 NTAPI
 375 KiInitializeMTRR(
 376     IN BOOLEAN FinalCpu
 377 );
 378
 379 VOID
 380 NTAPI
 381 KiAmdK6InitializeMTRR(
 382     VOID
 383 );
 384
 385 VOID
 386 NTAPI
 387 KiRestoreFastSyscallReturnState(
 388     VOID
 389 );
 390
 391 ULONG_PTR
 392 NTAPI
 393 Ki386EnableDE(
 394     IN ULONG_PTR Context
 395 );
 396
 397 ULONG_PTR
 398 NTAPI
 399 Ki386EnableFxsr(
 400     IN ULONG_PTR Context
 401 );
 402
 403 ULONG_PTR
 404 NTAPI
 405 Ki386EnableXMMIExceptions(
 406     IN ULONG_PTR Context
 407 );
 408
 409 BOOLEAN
 410 NTAPI
 411 VdmDispatchBop(
 412     IN PKTRAP_FRAME TrapFrame
 413 );
 414
 415 BOOLEAN
 416 FASTCALL
 417 KiVdmOpcodePrefix(
 418     IN PKTRAP_FRAME TrapFrame,
 419     IN ULONG Flags
 420 );
 421
 422 BOOLEAN
 423 FASTCALL
 424 Ki386HandleOpcodeV86(
 425     IN PKTRAP_FRAME TrapFrame
 426 );
 427
 428 DECLSPEC_NORETURN
 429 VOID
 430 FASTCALL
 431 KiEoiHelper(
 432     IN PKTRAP_FRAME TrapFrame
 433 );
 434
 435 VOID
 436 FASTCALL
 437 Ki386BiosCallReturnAddress(
 438     IN PKTRAP_FRAME TrapFrame
 439 );
 440
 441 ULONG_PTR
 442 FASTCALL
 443 KiExitV86Mode(
 444     IN PKTRAP_FRAME TrapFrame
 445 );
 446
 447 DECLSPEC_NORETURN
 448 VOID
 449 NTAPI
 450 KiDispatchExceptionFromTrapFrame(
 451     IN NTSTATUS Code,
 452     IN ULONG_PTR Address,
 453     IN ULONG ParameterCount,
 454     IN ULONG_PTR Parameter1,
 455     IN ULONG_PTR Parameter2,
 456     IN ULONG_PTR Parameter3,
 457     IN PKTRAP_FRAME TrapFrame
 458 );
 459
 460 //
 461 // Global x86 only Kernel data
 462 //
 463 extern PVOID Ki386IopmSaveArea;
 464 extern ULONG KeI386EFlagsAndMaskV86;
 465 extern ULONG KeI386EFlagsOrMaskV86;
 466 extern BOOLEAN KeI386VirtualIntExtensions;
 467 extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR+1];
 468 extern KDESCRIPTOR KiIdtDescriptor;
 469 extern BOOLEAN KiI386PentiumLockErrataPresent;
 470 extern ULONG KeI386NpxPresent;
 471 extern ULONG KeI386XMMIPresent;
 472 extern ULONG KeI386FxsrPresent;
 473 extern ULONG KiMXCsrMask;
 474 extern ULONG KeI386CpuType;
 475 extern ULONG KeI386CpuStep;
 476 extern ULONG Ke386CacheAlignment;
 477 extern ULONG KiFastSystemCallDisable;
 478 extern UCHAR KiDebugRegisterTrapOffsets[9];
 479 extern UCHAR KiDebugRegisterContextOffsets[9];
 480 extern DECLSPEC_NORETURN VOID __cdecl KiTrap02(VOID);
 481 extern VOID __cdecl KiTrap08(VOID);
 482 extern VOID __cdecl KiTrap13(VOID);
 483 extern VOID __cdecl KiFastCallEntry(VOID);
 484 extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
 485 extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID);
 486 extern VOID __cdecl CopyParams(VOID);
 487 extern VOID __cdecl ReadBatch(VOID);
 488 extern VOID __cdecl FrRestore(VOID);
 489 extern CHAR KiSystemCallExitBranch[];
 490 extern CHAR KiSystemCallExit[];
 491 extern CHAR KiSystemCallExit2[];
 492
 493 //
 494 // Trap Macros
 495 //
 496 #include "trap_x.h"
 497
 498 //
 499 // Returns a thread's FPU save area
 500 //
 501 PFX_SAVE_AREA
 502 FORCEINLINE
 503 KiGetThreadNpxArea(IN PKTHREAD Thread)
 504 {
 505     return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
 506 }
 507
 508 //
 509 // Sanitizes a selector
 510 //
 511 FORCEINLINE
 512 ULONG
 513 Ke386SanitizeSeg(IN ULONG Cs,
 514                 IN KPROCESSOR_MODE Mode)
 515 {
 516     //
 517     // Check if we're in kernel-mode, and force CPL 0 if so.
 518     // Otherwise, force CPL 3.
 519     //
 520     return ((Mode == KernelMode) ?
 521             (Cs & (0xFFFF & ~RPL_MASK)) :
 522             (RPL_MASK | (Cs & 0xFFFF)));
 523 }
 524
 525 //
 526 // Sanitizes EFLAGS
 527 //
 528 FORCEINLINE
 529 ULONG
 530 Ke386SanitizeFlags(IN ULONG Eflags,
 531                    IN KPROCESSOR_MODE Mode)
 532 {
 533     //
 534     // Check if we're in kernel-mode, and sanitize EFLAGS if so.
 535     // Otherwise, also force interrupt mask on.
 536     //
 537     return ((Mode == KernelMode) ?
 538             (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
 539             (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
 540 }
 541
 542 //
 543 // Gets a DR register from a CONTEXT structure
 544 //
 545 FORCEINLINE
 546 PVOID
 547 KiDrFromContext(IN ULONG Dr,
 548                 IN PCONTEXT Context)
 549 {
 550     return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]);
 551 }
 552
 553 //
 554 // Gets a DR register from a KTRAP_FRAME structure
 555 //
 556 FORCEINLINE
 557 PVOID*
 558 KiDrFromTrapFrame(IN ULONG Dr,
 559                   IN PKTRAP_FRAME TrapFrame)
 560 {
 561     return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]);
 562 }
 563
 564 //
 565 // Sanitizes a Debug Register
 566 //
 567 FORCEINLINE
 568 PVOID
 569 Ke386SanitizeDr(IN PVOID DrAddress,
 570                 IN KPROCESSOR_MODE Mode)
 571 {
 572     //
 573     // Check if we're in kernel-mode, and return the address directly if so.
 574     // Otherwise, make sure it's not inside the kernel-mode address space.
 575     // If it is, then clear the address.
 576     //
 577     return ((Mode == KernelMode) ? DrAddress :
 578             (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
 579 }
 580
 581 //
 582 // Exception with no arguments
 583 //
 584 VOID
 585 FORCEINLINE
 586 DECLSPEC_NORETURN
 587 KiDispatchException0Args(IN NTSTATUS Code,
 588                          IN ULONG_PTR Address,
 589                          IN PKTRAP_FRAME TrapFrame)
 590 {
 591     /* Helper for exceptions with no arguments */
 592     KiDispatchExceptionFromTrapFrame(Code, Address, 0, 0, 0, 0, TrapFrame);
 593 }
 594
 595 //
 596 // Exception with one argument
 597 //
 598 VOID
 599 FORCEINLINE
 600 DECLSPEC_NORETURN
 601 KiDispatchException1Args(IN NTSTATUS Code,
 602                          IN ULONG_PTR Address,
 603                          IN ULONG P1,
 604                          IN PKTRAP_FRAME TrapFrame)
 605 {
 606     /* Helper for exceptions with no arguments */
 607     KiDispatchExceptionFromTrapFrame(Code, Address, 1, P1, 0, 0, TrapFrame);
 608 }
 609
 610 //
 611 // Exception with two arguments
 612 //
 613 VOID
 614 FORCEINLINE
 615 DECLSPEC_NORETURN
 616 KiDispatchException2Args(IN NTSTATUS Code,
 617                          IN ULONG_PTR Address,
 618                          IN ULONG P1,
 619                          IN ULONG P2,
 620                          IN PKTRAP_FRAME TrapFrame)
 621 {
 622     /* Helper for exceptions with no arguments */
 623     KiDispatchExceptionFromTrapFrame(Code, Address, 2, P1, P2, 0, TrapFrame);
 624 }
 625
 626 //
 627 // Performs a system call
 628 //
 629
 630     /*
 631      * This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
 632      * and then calls the function associated with the system call.
 633      *
 634      * It's done in assembly for two reasons: we need to muck with the stack,
 635      * and the call itself restores the stack back for us. The only way to do
 636      * this in C is to do manual C handlers for every possible number of args on
 637      * the stack, and then have the handler issue a call by pointer. This is
 638      * wasteful since it'll basically push the values twice and require another
 639      * level of call indirection.
 640      *
 641      * The ARM kernel currently does this, but it should probably be changed
 642      * later to function like this as well.
 643      *
 644      */
 645 #ifdef __GNUC__
 646 NTSTATUS
 647 FORCEINLINE
 648 KiSystemCallTrampoline(IN PVOID Handler,
 649                        IN PVOID Arguments,
 650                        IN ULONG StackBytes)
 651 {
 652     NTSTATUS Result;
 653
 654     __asm__ __volatile__
 655     (
 656         "subl %1, %%esp\n"
 657         "movl %%esp, %%edi\n"
 658         "movl %2, %%esi\n"
 659         "shrl $2, %1\n"
 660         "rep movsd\n"
 661         "call *%3\n"
 662         "movl %%eax, %0\n"
 663         : "=r"(Result)
 664         : "c"(StackBytes),
 665           "d"(Arguments),
 666           "r"(Handler)
 667         : "%esp", "%esi", "%edi"
 668     );
 669     return Result;
 670 }
 671 #elif defined(_MSC_VER)
 672 NTSTATUS
 673 FORCEINLINE
 674 KiSystemCallTrampoline(IN PVOID Handler,
 675                        IN PVOID Arguments,
 676                        IN ULONG StackBytes)
 677 {
 678     __asm
 679     {
 680         mov ecx, StackBytes
 681         mov esi, Arguments
 682         mov eax, Handler
 683         sub esp, ecx
 684         mov edi, esp
 685         shr ecx, 2
 686         rep movsd
 687         call eax
 688     }
 689     /* Return with result in EAX */
 690 }
 691 #else
 692 #error Unknown Compiler
 693 #endif
 694
 695
 696 //
 697 // Checks for pending APCs
 698 //
 699 VOID
 700 FORCEINLINE
 701 KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
 702 {
 703     PKTHREAD Thread;
 704     KIRQL OldIrql;
 705
 706     /* Check for V8086 or user-mode trap */
 707     if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
 708     {
 709         /* Get the thread */
 710         Thread = KeGetCurrentThread();
 711         while (TRUE)
 712         {
 713             /* Turn off the alerted state for kernel mode */
 714             Thread->Alerted[KernelMode] = FALSE;
 715
 716             /* Are there pending user APCs? */
 717             if (!Thread->ApcState.UserApcPending) break;
 718
 719             /* Raise to APC level and enable interrupts */
 720             OldIrql = KfRaiseIrql(APC_LEVEL);
 721             _enable();
 722
 723             /* Deliver APCs */
 724             KiDeliverApc(UserMode, NULL, TrapFrame);
 725
 726             /* Restore IRQL and disable interrupts once again */
 727             KfLowerIrql(OldIrql);
 728             _disable();
 729         }
 730     }
 731 }
 732
 733 //
 734 // Converts a base thread to a GUI thread
 735 //
 736 #ifdef __GNUC__
 737 NTSTATUS
 738 FORCEINLINE
 739 KiConvertToGuiThread(VOID)
 740 {
 741     NTSTATUS Result;
 742     PVOID StackFrame;
 743
 744     /*
 745      * Converting to a GUI thread safely updates ESP in-place as well as the
 746      * current Thread->TrapFrame and EBP when KeSwitchKernelStack is called.
 747      *
 748      * However, PsConvertToGuiThread "helpfully" restores EBP to the original
 749      * caller's value, since it is considered a nonvolatile register. As such,
 750      * as soon as we're back after the conversion and we try to store the result
 751      * which will probably be in some stack variable (EBP-based), we'll crash as
 752      * we are touching the de-allocated non-expanded stack.
 753      *
 754      * Thus we need a way to update our EBP before EBP is touched, and the only
 755      * way to guarantee this is to do the call itself in assembly, use the EAX
 756      * register to store the result, fixup EBP, and then let the C code continue
 757      * on its merry way.
 758      *
 759      */
 760     __asm__ __volatile__
 761     (
 762         "movl %%ebp, %1\n\t"
 763         "subl %%esp, %1\n\t"
 764         "call _PsConvertToGuiThread@0\n\t"
 765         "addl %%esp, %1\n\t"
 766         "movl %1, %%ebp"
 767         : "=a"(Result), "=r"(StackFrame)
 768         :
 769         : "%esp", "%ecx", "%edx", "memory"
 770     );
 771     return Result;
 772 }
 773 #elif defined(_MSC_VER)
 774 NTSTATUS
 775 NTAPI
 776 KiConvertToGuiThread(VOID);
 777 #else
 778 #error Unknown Compiler
 779 #endif
 780
 781 //
 782 // Switches from boot loader to initial kernel stack
 783 //
 784 VOID
 785 FORCEINLINE
 786 KiSwitchToBootStack(IN ULONG_PTR InitialStack)
 787 {
 788     /* We have to switch to a new stack before continuing kernel initialization */
 789 #ifdef __GNUC__
 790     __asm__
 791     (
 792         "movl %0, %%esp\n"
 793         "subl %1, %%esp\n"
 794         "pushl %2\n"
 795         "jmp _KiSystemStartupBootStack@0\n"
 796         :
 797         : "c"(InitialStack),
 798           "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
 799           "i"(CR0_EM | CR0_TS | CR0_MP)
 800         : "%esp"
 801     );
 802 #elif defined(_MSC_VER)
 803     VOID NTAPI KiSystemStartupBootStack(VOID);
 804     __asm
 805     {
 806         mov esp, InitialStack
 807         sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
 808         push (CR0_EM | CR0_TS | CR0_MP)
 809         jmp KiSystemStartupBootStack
 810     }
 811 #else
 812 #error Unknown Compiler
 813 #endif
 814 }
 815
 816 //
 817 // Emits the iret instruction for C code
 818 //
 819 DECLSPEC_NORETURN
 820 VOID
 821 FORCEINLINE
 822 KiIret(VOID)
 823 {
 824 #if defined(__GNUC__)
 825     __asm__ __volatile__
 826     (
 827         "iret\n"
 828     );
 829 #elif defined(_MSC_VER)
 830     __asm
 831     {
 832         iretd
 833     }
 834 #else
 835 #error Unsupported compiler
 836 #endif
 837     UNREACHABLE;
 838 }
 839
 840 //
 841 // Normally this is done by the HAL, but on x86 as an optimization, the kernel
 842 // initiates the end by calling back into the HAL and exiting the trap here.
 843 //
 844 VOID
 845 FORCEINLINE
 846 KiEndInterrupt(IN KIRQL Irql,
 847                IN PKTRAP_FRAME TrapFrame)
 848 {
 849     /* Disable interrupts and end the interrupt */
 850     _disable();
 851     HalEndSystemInterrupt(Irql, TrapFrame);
 852
 853     /* Exit the interrupt */
 854     KiEoiHelper(TrapFrame);
 855 }
 856
 857 //
 858 // PERF Code
 859 //
 860 VOID
 861 FORCEINLINE
 862 Ki386PerfEnd(VOID)
 863 {
 864     extern ULONGLONG BootCyclesEnd, BootCycles;
 865     BootCyclesEnd = __rdtsc();
 866     DbgPrint("Boot took %I64d cycles!\n", BootCyclesEnd - BootCycles);
 867     DbgPrint("Interrupts: %d System Calls: %d Context Switches: %d\n",
 868              KeGetCurrentPrcb()->InterruptCount,
 869              KeGetCurrentPrcb()->KeSystemCalls,
 870              KeGetContextSwitches(KeGetCurrentPrcb()));
 871 }
 872
 873 FORCEINLINE
 874 PULONG
 875 KiGetUserModeStackAddress(void)
 876 {
 877     return &(KeGetCurrentThread()->TrapFrame->HardwareEsp);
 878 }
 879
 880 #endif