ntoskrnl/include/internal/i386/asmmacro.S

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS Kernel
   4  * FILE:            ntoskrnl/include/i386/asmmacro.S
   5  * PURPOSE:         Assembly Macros for Spinlocks and common Trap Code
   6  * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
   7  */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include <ndk/asm.h>
  12
  13 // Arguments for TRAP_EPILOG
  14 #define FromSystemCall              1
  15 #define DoRestorePreviousMode       1
  16 #define DoRestoreEverything         1
  17 #define DoRestoreSegments           1
  18 #define DoRestoreVolatiles          1
  19 #define DoPushFakeErrorCode         1
  20 #define DoFixupV86                  1
  21 #define DoFixupAbios                1
  22 #define NotFromSystemCall           0
  23 #define DoNotRestorePreviousMode    0
  24 #define DoNotRestoreEverything      0
  25 #define DoNotRestoreSegments        0
  26 #define DoNotRestoreVolatiles       0
  27 #define DoNotPushFakeErrorCode      0
  28 #define DoNotFixupV86               0
  29 #define DoNotFixupAbios             0
  30
  31 // Arguments for idt
  32 #define INT_32_DPL0                 0x8E00
  33 #define INT_32_DPL3                 0xEE00
  34
  35 .intel_syntax noprefix
  36
  37 //
  38 // These macros are inlined equivalents of KiAcquire/ReleaseSpinlock, that is,
  39 // they will not be compiled into non-SMP builds. Usage is as follows:
  40 //
  41 // .BeginYourFunction
  42 //      mov reg, lockaddr
  43 //      ACQUIRE_SPINLOCK(reg, .spin)
  44 //      <thread-safe code here>
  45 //      RELEASE_SPINLOCK(reg)
  46 //      <misc code here>
  47 //  retn
  48 //  #IFDEF CONFIG_SMP
  49 //  .spin
  50 //      <any necessary steps to be able to jump back safely>
  51 /       SPIN_ON_LOCK(reg, .BeginYourFunction)
  52 //  #ENDIF
  53 //
  54 #ifdef CONFIG_SMP
  55 #define LOCK lock
  56 #define ACQUIRE_SPINLOCK(x, y) \
  57     lock bts dword ptr [x], 0; \
  58     jb y
  59 #define RELEASE_SPINLOCK(x) mov byte ptr [x], 0
  60 #define SPIN_ON_LOCK(x, y) \
  61 1: \
  62     test dword ptr [x], 1; \
  63     jz y; \
  64     pause; \
  65     jmp 1b
  66 #else
  67 #define LOCK
  68 #define ACQUIRE_SPINLOCK(x, y)
  69 #define RELEASE_SPINLOCK(x)
  70 #endif
  71
  72 //
  73 // @name UNHANDLED_PATH
  74 //
  75 // This macro TODO
  76 //
  77 // @param None
  78 //
  79 // @remark None.
  80 //
  81 .macro UNHANDLED_PATH
  82     /* Get EIP */
  83     call $+5
  84     pop eax
  85
  86     /* Print debug message */
  87     push eax
  88     push offset _UnhandledMsg
  89     call _DbgPrint
  90     add esp, 8
  91
  92     /* Loop indefinitely */
  93     jmp $
  94 .endm
  95
  96 //
  97 // @name IDT
  98 //
  99 // This macro creates an IDT entry for the given handler
 100 //
 101 // @param Handler
 102 //        Pointer to the IDT handler
 103 //
 104 // @param Bits
 105 //        Descriptor Bits to associate
 106 //
 107 // @remark None.
 108 //
 109 .macro idt Handler, Bits
 110     .long \Handler
 111     .short \Bits
 112     .short KGDT_R0_CODE
 113 .endm
 114
 115 //
 116 // @name GENERATE_IDT_STUB
 117 //
 118 // This macro creates an IDT entry for an unexpected interrupt handler.
 119 //
 120 // @param None.
 121 //
 122 // @remark None.
 123 //
 124 .macro GENERATE_IDT_STUB Number
 125 idt _KiUnexpectedInterrupt&Number, INT_32_DPL0
 126 .endm
 127
 128 //
 129 // @name GENERATE_IDT_STUBS
 130 //
 131 // This macro creates unexpected interrupt IDT entries.
 132 //
 133 // @param None.
 134 //
 135 // @remark None.
 136 //
 137 .altmacro
 138 .macro GENERATE_IDT_STUBS
 139 .set i, 0
 140 .rept 208
 141     GENERATE_IDT_STUB %i
 142     .set i, i + 1
 143 .endr
 144 .endm
 145
 146 //
 147 // @name GENERATE_INT_HANDLER
 148 //
 149 // This macro creates an unexpected interrupt handler.
 150 //
 151 // @param None.
 152 //
 153 // @remark None.
 154 //
 155 .macro GENERATE_INT_HANDLER Number
 156 .func KiUnexpectedInterrupt&Number
 157 _KiUnexpectedInterrupt&Number:
 158     push PRIMARY_VECTOR_BASE + Number
 159     jmp _KiEndUnexpectedRange@0
 160 .endfunc
 161 .endm
 162
 163 //
 164 // @name GENERATE_INT_HANDLERS
 165 //
 166 // This macro creates the unexpected interrupt handlers.
 167 //
 168 // @param None.
 169 //
 170 // @remark None.
 171 //
 172 .altmacro
 173 .macro GENERATE_INT_HANDLERS
 174 .set i, 0
 175 .rept 208
 176     GENERATE_INT_HANDLER %i
 177     .set i, i + 1
 178 .endr
 179 .endm
 180
 181 //
 182 // @name INVALID_V86_OPCODE
 183 //
 184 // This macro creates one or more entries for unhandled V86 Opcodes
 185 // in the V86 Opcode Table.
 186 //
 187 // @param count.
 188 //        Number of entries to generate.
 189 //
 190 // @remark None.
 191 //
 192 .macro INVALID_V86_OPCODE count
 193     .rept \count
 194         .byte 0
 195     .endr
 196 .endm
 197
 198 //
 199 // @name INVALID_V86_OPCODE
 200 //
 201 // This macro prints out visible message and hangs the computer.
 202 //
 203 // @param None.
 204 //
 205 // @remark Temporary debugging use.
 206 //
 207 .macro UNHANDLED_V86_OPCODE
 208     /* Print debug message, breakpoint and freeze */
 209     push ecx
 210     push offset V86DebugMsg
 211     call _DbgPrint
 212     add esp, 8
 213     jmp $
 214 .endm
 215
 216 //
 217 // @name TRAP_FIXUPS
 218 //
 219 // This macro contains out-of-line code for various Trap Frame Fixups, such as:
 220 //
 221 //  - DR Fixup: Loads and restores DR registers.
 222 //  - V86 Fixup: Loads and restores V86 segments.
 223 //  - ABIOS Fixup: Loads and restores the ABIOS state and stack.
 224 //
 225 // @param None.
 226 //
 227 // @remark ebp = PKTRAP_FRAME
 228 //
 229 .macro TRAP_FIXUPS Label, EndLabel, V86Fix, AbiosFix
 230 Dr_&Label:
 231
 232     /* Check if this was V86 mode */
 233     test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
 234     jnz 2f
 235
 236     /* Check if it was user mode */
 237     test byte ptr [ebp+KTRAP_FRAME_CS], MODE_MASK
 238     jz Dr_&EndLabel
 239
 240 2:
 241     /* Get DR0, 1, 2 */
 242     mov ebx, dr0
 243     mov ecx, dr1
 244     mov edi, dr2
 245
 246     /* Save them */
 247     mov [ebp+KTRAP_FRAME_DR0], ebx
 248     mov [ebp+KTRAP_FRAME_DR1], ecx
 249     mov [ebp+KTRAP_FRAME_DR2], edi
 250
 251     /* Get DR3, 6, 7 */
 252     mov ebx, dr3
 253     mov ecx, dr6
 254     mov edi, dr7
 255
 256     /* Save them */
 257     mov [ebp+KTRAP_FRAME_DR3], ebx
 258     mov [ebp+KTRAP_FRAME_DR6], ecx
 259     mov [ebp+KTRAP_FRAME_DR7], edi
 260
 261     /* Clear DR7 */
 262     xor ebx, ebx
 263     mov dr7, ebx
 264
 265     /* Get the PRCB */
 266     mov edi, fs:[KPCR_PRCB]
 267
 268     /* Get DR0, 1 */
 269     mov ebx, [edi+KPRCB_DR0]
 270     mov ecx, [edi+KPRCB_DR1]
 271
 272     /* Set them */
 273     mov dr0, ebx
 274     mov dr1, ecx
 275
 276     /* Get DR2, 3 */
 277     mov ebx, [edi+KPRCB_DR2]
 278     mov ecx, [edi+KPRCB_DR3]
 279
 280     /* Set them */
 281     mov dr2, ebx
 282     mov dr3, ecx
 283
 284     /* Get DR6, 7 */
 285     mov ebx, [edi+KPRCB_DR6]
 286     mov ecx, [edi+KPRCB_DR7]
 287
 288     /* Set them */
 289     mov dr6, ebx
 290     mov dr7, ecx
 291     jmp Dr_&EndLabel
 292
 293 .if \AbiosFix
 294 Abios_&Label:
 295     UNHANDLED_PATH
 296 .endif
 297
 298 .if \V86Fix
 299 V86_&Label:
 300
 301     /* Get V86 segment registers */
 302     mov eax, [ebp+KTRAP_FRAME_V86_FS]
 303     mov ebx, [ebp+KTRAP_FRAME_V86_GS]
 304     mov ecx, [ebp+KTRAP_FRAME_V86_ES]
 305     mov edx, [ebp+KTRAP_FRAME_V86_DS]
 306
 307     /* Restore them into Protected Mode trap frame */
 308     mov [ebp+KTRAP_FRAME_FS], ax
 309     mov [ebp+KTRAP_FRAME_GS], bx
 310     mov [ebp+KTRAP_FRAME_ES], cx
 311     mov [ebp+KTRAP_FRAME_DS], dx
 312
 313     /* Go back to mainline code */
 314     jmp V86_&EndLabel
 315 .endif
 316 .endm
 317
 318 //
 319 // @name SET_TF_DEBUG_HEADER
 320 //
 321 // This macro sets up the debug header in the trap frame.
 322 //
 323 // @param None.
 324 //
 325 // @remark ebp = PKTRAP_FRAME.
 326 //         edi/ebx = Have been saved and can be used.
 327 //
 328 .macro SET_TF_DEBUG_HEADER
 329     /* Get the Debug Trap Frame EBP/EIP */
 330     mov ebx, [ebp+KTRAP_FRAME_EBP]
 331     mov edi, [ebp+KTRAP_FRAME_EIP]
 332
 333     /* Write the debug data */
 334     mov [ebp+KTRAP_FRAME_DEBUGPOINTER], edx
 335     mov dword ptr [ebp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
 336     mov [ebp+KTRAP_FRAME_DEBUGEBP], ebx
 337     mov [ebp+KTRAP_FRAME_DEBUGEIP], edi
 338 .endm
 339
 340 //
 341 // @name CHECK_FOR_APC_DELIVER
 342 //
 343 // This macro checks if the trapframe indicates a return to user-mode,
 344 // and, if so, checks if user-mode APCs should be delivered.
 345 //
 346 // @param PreserveEax
 347 //        Determines if EAX should be preserved. Implies that the segment
 348 //        registers will also be saved.
 349 //
 350 // @remark ebp = PKTRAP_FRAME.
 351 //         ebx = Saved and will be used.
 352 //
 353 .macro CHECK_FOR_APC_DELIVER PreserveEax
 354     /* Check for V86 mode */
 355     test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
 356     jnz 1f
 357
 358     /* Deliver APCs only if we were called from user mode */
 359     test byte ptr [ebp+KTRAP_FRAME_CS], 1
 360     je 2f
 361
 362     /* Get the current thread */
 363 1:
 364     mov ebx, PCR[KPCR_CURRENT_THREAD]
 365
 366     /* Make it non-alerted */
 367     mov byte ptr [ebx+KTHREAD_ALERTED], 0
 368
 369     /* And only if any are actually pending */
 370     cmp byte ptr [ebx+KTHREAD_PENDING_USER_APC], 0
 371     je 2f
 372
 373     /* Save pointer to Trap Frame */
 374     mov ebx, ebp
 375
 376 .if \PreserveEax
 377     /* Save some stuff that raising IRQL will kill */
 378     mov [ebx+KTRAP_FRAME_EAX], eax
 379     mov dword ptr [ebx+KTRAP_FRAME_FS], KGDT_R3_TEB + RPL_MASK
 380     mov dword ptr [ebx+KTRAP_FRAME_DS], KGDT_R3_DATA + RPL_MASK
 381     mov dword ptr [ebx+KTRAP_FRAME_ES], KGDT_R3_DATA + RPL_MASK
 382     mov dword ptr [ebx+KTRAP_FRAME_GS], 0
 383 .endif
 384
 385     /* Raise IRQL to APC_LEVEL */
 386     mov ecx, 1
 387     call @KfRaiseIrql@4
 388
 389     /* Save old IRQL */
 390     push eax
 391
 392     /* Deliver APCs */
 393     sti
 394     push ebx
 395     push 0
 396     push UserMode
 397     call _KiDeliverApc@12
 398
 399     /* Return to old IRQL */
 400     pop ecx
 401     call @KfLowerIrql@4
 402
 403     /* Restore EAX (only in volatile case) */
 404 .if \PreserveEax
 405     mov eax, [ebx+KTRAP_FRAME_EAX]
 406 .endif
 407     cli
 408     jmp 1b
 409 2:
 410 .endm
 411
 412 //
 413 // @name TRAP_PROLOG
 414 //
 415 // This macro creates a standard trap entry prologue.
 416 // It should be used for entry into any kernel trap (KiTrapXx), but not for
 417 // system calls, which require special handling.
 418 //
 419 // @param Label
 420 //        Identifying name of the caller function; will be used to append
 421 //        to the name V86 and DR helper functions, which must already exist.
 422 //
 423 // @remark Use as follows:
 424 //          _KiTrap00:
 425 //              /* Push fake error code */
 426 //              push 0
 427 //
 428 //              /* Enter common prologue */
 429 //              TRAP_PROLOG(0)
 430 //
 431 //              /* Handle trap */
 432 //              <Your Trap Code Here>
 433 //
 434 .macro TRAP_PROLOG Label EndLabel
 435     /* Just to be safe, clear out the HIWORD, since it's reserved */
 436     mov word ptr [esp+2], 0
 437
 438     /* Save the non-volatiles */
 439     push ebp
 440     push ebx
 441     push esi
 442     push edi
 443
 444     /* Save FS and set it to PCR */
 445     push fs
 446     mov ebx, KGDT_R0_PCR
 447     .byte 0x66
 448     mov fs, bx
 449
 450     /* Save exception list and bogus previous mode */
 451     push fs:[KPCR_EXCEPTION_LIST]
 452     push -1
 453
 454     /* Save volatiles and segment registers */
 455     push eax
 456     push ecx
 457     push edx
 458     push ds
 459     push es
 460     push gs
 461
 462     /* Set the R3 data segment */
 463     mov ax, KGDT_R3_DATA + RPL_MASK
 464
 465     /* Skip debug registers and debug stuff */
 466     sub esp, 0x30
 467
 468     /* Load the segment registers */
 469     .byte 0x66
 470     mov ds, ax
 471     .byte 0x66
 472     mov es, ax
 473
 474     /* Check if this interrupt happened in 16-bit mode */
 475     cmp esp, 0x10000
 476     jb _Ki16BitStackException
 477
 478     /* Set up frame */
 479     mov ebp, esp
 480
 481     /* Check if this was from V86 Mode */
 482     test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
 483     jnz V86_&Label
 484
 485 V86_&EndLabel:
 486     /* Get current thread */
 487     mov ecx, fs:[KPCR_CURRENT_THREAD]
 488     cld
 489
 490     /* Flush DR7 */
 491     and dword ptr [ebp+KTRAP_FRAME_DR7], 0
 492
 493     /* Check if the thread was being debugged */
 494     test byte ptr [ecx+KTHREAD_DEBUG_ACTIVE], 0xFF
 495     jnz Dr_&Label
 496
 497     /* Set the Trap Frame Debug Header */
 498 Dr_&EndLabel:
 499     SET_TF_DEBUG_HEADER
 500 .endm
 501
 502 //
 503 // @name INT_PROLOG
 504 //
 505 // This macro creates a standard interrupt entry prologue.
 506 // It should be used for entry into any interrupt, including software.
 507 //
 508 // @param Label
 509 //        Identifying name of the caller function; will be used to append
 510 //        to the name V86, ABIOS and DR helper functions, which must exist.
 511 //
 512 // @remark For software interrupts, make sure that a fake INT stack is created.
 513 //
 514 .macro INT_PROLOG Label EndLabel FakeErrorCode
 515
 516 .if \FakeErrorCode
 517     /* Save fake error code */
 518     push esp
 519 .endif
 520
 521     /* Save the non-volatiles */
 522     push ebp
 523     push ebx
 524     push esi
 525     push edi
 526
 527     /* Skip debug registers and other stuff */
 528     sub esp, 0x54
 529
 530     /* Set up frame */
 531     mov ebp, esp
 532
 533     /* Save volatiles */
 534     mov [esp+KTRAP_FRAME_EAX], eax
 535     mov [esp+KTRAP_FRAME_ECX], ecx
 536     mov [esp+KTRAP_FRAME_EDX], edx
 537     mov dword ptr [esp+KTRAP_FRAME_PREVIOUS_MODE], -1
 538
 539     /* Check if this was from V86 Mode */
 540     test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
 541     jnz V86_&Label
 542
 543     /* Check if this was kernel mode */
 544 V86_&EndLabel:
 545     cmp word ptr [esp+KTRAP_FRAME_CS], KGDT_R0_CODE
 546     jz 1f
 547
 548     /* Set segments */
 549     mov word ptr [esp+KTRAP_FRAME_FS], fs
 550     mov word ptr [esp+KTRAP_FRAME_DS], ds
 551     mov word ptr [esp+KTRAP_FRAME_ES], es
 552     mov [esp+KTRAP_FRAME_GS], gs
 553
 554     /* Load the segment registers (use OVERRIDE (0x66)) */
 555     mov ebx, KGDT_R0_PCR
 556     mov eax, KGDT_R3_DATA | RPL_MASK
 557     .byte 0x66
 558     mov fs, bx
 559     .byte 0x66
 560     mov ds, ax
 561     .byte 0x66
 562     mov es, ax
 563
 564 1:
 565     /* Get the previous exception list */
 566     mov ebx, fs:[KPCR_EXCEPTION_LIST]
 567
 568     /* Set the exception handler chain terminator */
 569     mov dword ptr fs:[KPCR_EXCEPTION_LIST], -1
 570
 571     /* Save the previous exception list */
 572     mov [esp+KTRAP_FRAME_EXCEPTION_LIST], ebx
 573
 574 .ifeq \FakeErrorCode
 575     /* Setup the 16-bit stack */
 576     lea eax, [esp+KTRAP_FRAME_ERROR_CODE]
 577     lea ecx, [esp+KTRAP_FRAME_EIP]
 578     mov ebx, ss:[eax]
 579     mov ss:[eax], ecx
 580 .endif
 581
 582     /* Check if this is the ABIOS stack */
 583     /* cmp esp, 0x10000*/
 584     /* jb Abios_Label*/
 585
 586     /* Delete error code */
 587     and dword ptr [esp+KTRAP_FRAME_ERROR_CODE], 0
 588
 589     /* Get the current thread and clear direction flag */
 590     mov ecx, PCR[KPCR_CURRENT_THREAD]
 591     cld
 592
 593     /* Flush DR7 */
 594     and dword ptr [ebp+KTRAP_FRAME_DR7], 0
 595
 596     /* Check if the thread was being debugged */
 597     test byte ptr [ecx+KTHREAD_DEBUG_ACTIVE], 0xFF
 598
 599 .ifeq \FakeErrorCode
 600     /* Push parameter */
 601     push ebx
 602 .endif
 603
 604     /* Save DR registers if needed */
 605     jnz Dr_&Label
 606
 607     /* Set the trap frame debug header */
 608 Dr_&EndLabel:
 609     SET_TF_DEBUG_HEADER
 610 .endm
 611
 612 //
 613 // @name SYSCALL_PROLOG
 614 //
 615 // This macro creates a system call entry prologue.
 616 // It should be used for entry into any fast-system call (KiGetTickCount,
 617 // KiCallbackReturn, KiRaiseAssertion) and the generic system call handler
 618 // (KiSystemService)
 619 //
 620 // @param Label
 621 //        Unique label identifying the name of the caller function; will be
 622 //        used to append to the name of the DR helper function, which must
 623 //        already exist.
 624 //
 625 // @remark None.
 626 //
 627 .macro SYSCALL_PROLOG Label EndLabel
 628     /* Create a trap frame */
 629     push 0
 630     push ebp
 631     push ebx
 632     push esi
 633     push edi
 634     push fs
 635
 636     /* Load PCR Selector into fs */
 637     mov ebx, KGDT_R0_PCR
 638     .byte 0x66
 639     mov fs, bx
 640
 641     /* Get a pointer to the current thread */
 642     mov esi, PCR[KPCR_CURRENT_THREAD]
 643
 644     /* Save the previous exception list */
 645     push PCR[KPCR_EXCEPTION_LIST]
 646
 647     /* Set the exception handler chain terminator */
 648     mov dword ptr PCR[KPCR_EXCEPTION_LIST], -1
 649
 650     /* Save the old previous mode */
 651     push [esi+KTHREAD_PREVIOUS_MODE]
 652
 653     /* Skip the other registers */
 654     sub esp, 0x48
 655
 656     /* Set the new previous mode based on the saved CS selector */
 657     mov ebx, [esp+0x6C]
 658     and ebx, 1
 659     mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], bl
 660
 661     /* Go on the Kernel stack frame */
 662     mov ebp, esp
 663
 664     /* Save the old trap frame pointer where EDX would be saved */
 665     mov ebx, [esi+KTHREAD_TRAP_FRAME]
 666     mov [ebp+KTRAP_FRAME_EDX], ebx
 667
 668     /* Flush DR7 */
 669     and dword ptr [ebp+KTRAP_FRAME_DR7], 0
 670
 671     /* Check if the thread was being debugged */
 672     test byte ptr [esi+KTHREAD_DEBUG_ACTIVE], 0xFF
 673
 674     /* Set the thread's trap frame and clear direction flag */
 675     mov [esi+KTHREAD_TRAP_FRAME], ebp
 676     cld
 677
 678     /* Save DR registers if needed */
 679     jnz Dr_&Label
 680
 681     /* Set the trap frame debug header */
 682 Dr_&EndLabel:
 683     SET_TF_DEBUG_HEADER
 684
 685     /* Enable interrupts */
 686     sti
 687 .endm
 688
 689 //
 690 // @name FASTCALL_PROLOG
 691 //
 692 // TODO
 693 //
 694 // @param Label
 695 //        Unique label identifying the name of the caller function; will be
 696 //        used to append to the name of the DR helper function, which must
 697 //        already exist.
 698 //
 699 // @remark None.
 700 //
 701 .macro FASTCALL_PROLOG Label EndLabel
 702
 703     /* Set user selector */
 704     mov ecx, KGDT_R3_DATA | RPL_MASK
 705
 706     /* Set FS to PCR */
 707     push KGDT_R0_PCR
 708     pop fs
 709
 710     /* Set DS/ES to User Selector */
 711     mov ds, cx
 712     mov es, cx
 713
 714     /* Set the current stack to Kernel Stack */
 715     mov ecx, PCR[KPCR_TSS]
 716     mov esp, [ecx+KTSS_ESP0]
 717
 718     /* Set up a fake INT Stack. */
 719     push KGDT_R3_DATA + RPL_MASK
 720     push edx                            /* Ring 3 SS:ESP */
 721     pushf                               /* Ring 3 EFLAGS */
 722     push 2                              /* Ring 0 EFLAGS */
 723     add edx, 8                          /* Skip user parameter list */
 724     popf                                /* Set our EFLAGS */
 725     or dword ptr [esp], EFLAGS_INTERRUPT_MASK   /* Re-enable IRQs in EFLAGS, to fake INT */
 726     push KGDT_R3_CODE + RPL_MASK
 727     push dword ptr ds:KUSER_SHARED_SYSCALL_RET
 728
 729     /* Setup the Trap Frame stack */
 730     push 0
 731     push ebp
 732     push ebx
 733     push esi
 734     push edi
 735     push KGDT_R3_TEB + RPL_MASK
 736
 737     /* Save pointer to our PCR */
 738     mov ebx, PCR[KPCR_SELF]
 739
 740     /* Get a pointer to the current thread */
 741     mov esi, [ebx+KPCR_CURRENT_THREAD]
 742
 743     /* Set the exception handler chain terminator */
 744     push [ebx+KPCR_EXCEPTION_LIST]
 745     mov dword ptr [ebx+KPCR_EXCEPTION_LIST], -1
 746
 747     /* Use the thread's stack */
 748     mov ebp, [esi+KTHREAD_INITIAL_STACK]
 749
 750     /* Push previous mode */
 751     push UserMode
 752
 753     /* Skip the other registers */
 754     sub esp, 0x48
 755
 756     /* Make space for us on the stack */
 757     sub ebp, 0x29C
 758
 759     /* Write the previous mode */
 760     mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], UserMode
 761
 762     /* Sanity check */
 763     cmp ebp, esp
 764     jnz BadStack
 765
 766     /* Flush DR7 */
 767     and dword ptr [ebp+KTRAP_FRAME_DR7], 0
 768
 769     /* Check if the thread was being debugged */
 770     test byte ptr [esi+KTHREAD_DEBUG_ACTIVE], 0xFF
 771
 772     /* Set the thread's trap frame */
 773     mov [esi+KTHREAD_TRAP_FRAME], ebp
 774
 775     /* Save DR registers if needed */
 776     jnz Dr_&Label
 777
 778     /* Set the trap frame debug header */
 779 Dr_&EndLabel:
 780     SET_TF_DEBUG_HEADER
 781
 782     /* Enable interrupts */
 783     sti
 784 .endm
 785
 786 //
 787 // @name V86_TRAP_PROLOG
 788 //
 789 // This macro creates a V86 Trap entry prologue.
 790 // It should be used for entry into any fast-system call (KiGetTickCount,
 791 // KiCallbackReturn, KiRaiseAssertion) and the generic system call handler
 792 // (KiSystemService)
 793 //
 794 // @param Label
 795 //        Unique label identifying the name of the caller function; will be
 796 //        used to append to the name of the DR helper function, which must
 797 //        already exist.
 798 //
 799 // @remark None.
 800 //
 801 .macro V86_TRAP_PROLOG Label EndLabel
 802
 803     /* Skip everything to the error code */
 804     sub esp, KTRAP_FRAME_ERROR_CODE
 805
 806     /* Clear the error code */
 807     mov word ptr [esp+KTRAP_FRAME_ERROR_CODE+2], 0
 808
 809     /* Save the registers we'll trample */
 810     mov [esp+KTRAP_FRAME_EBX], ebx
 811     mov [esp+KTRAP_FRAME_EAX], eax
 812     mov [esp+KTRAP_FRAME_EBP], ebp
 813     mov [esp+KTRAP_FRAME_ESI], esi
 814     mov [esp+KTRAP_FRAME_EDI], edi
 815
 816     /* Save PCR and Ring 3 segments */
 817     mov ebx, KGDT_R0_PCR
 818     mov eax, KGDT_R3_DATA + RPL_MASK
 819
 820     /* Save ECX and EDX too */
 821     mov [esp+KTRAP_FRAME_ECX], ecx
 822     mov [esp+KTRAP_FRAME_EDX], edx
 823
 824     /* Set debugging markers */
 825     mov dword ptr [esp+KTRAP_FRAME_PREVIOUS_MODE], -1
 826     mov dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
 827
 828     /* Now set segments (use OVERRIDE, 0x66) */
 829     .byte 0x66
 830     mov fs, bx
 831     .byte 0x66
 832     mov ds, ax
 833     .byte 0x66
 834     mov es, ax
 835
 836     /* Set the trap frame in the stack and clear the direction flag */
 837     mov ebp, esp
 838     cld
 839
 840     /* Save the exception list */
 841     mov eax, fs:[KPCR_EXCEPTION_LIST]
 842     mov [esp+KTRAP_FRAME_EXCEPTION_LIST], eax
 843
 844     /* Check if we need debugging */
 845     mov eax, dr7
 846     test eax, ~DR7_RESERVED_MASK
 847     mov [esp+KTRAP_FRAME_DR7], eax
 848     jnz Dr_&Label
 849
 850 Dr_&EndLabel:
 851 .endm
 852
 853 //
 854 // @name V86_TRAP_EPILOG
 855 //
 856 // This macro creates an epilogue for leaving V86 traps
 857 //
 858 // @param None.
 859 //
 860 // @remark None.
 861 //
 862 .macro V86_TRAP_EPILOG
 863
 864     /* Get the current thread and make it unalerted */
 865 ExitBegin:
 866     mov ebx, PCR[KPCR_CURRENT_THREAD]
 867     mov byte ptr [ebx+KTHREAD_ALERTED], 0
 868
 869     /* Check if it has User-mode APCs pending */
 870     cmp byte ptr [ebx+KTHREAD_PENDING_USER_APC], 0
 871     jne PendingUserApc
 872
 873     /* It doesn't, pop the frame */
 874     add esp, KTRAP_FRAME_EDX
 875     pop edx
 876     pop ecx
 877     pop eax
 878
 879     /* Check if DR registers should be restored */
 880     test dword ptr [ebp+KTRAP_FRAME_DR7], ~DR7_RESERVED_MASK
 881     jnz V86DebugRestore
 882
 883     /* Finish popping the rest of the frame, and return to P-mode */
 884 V86DebugContinue:
 885     add esp, 12
 886     pop edi
 887     pop esi
 888     pop ebx
 889     pop ebp
 890     add esp, 4
 891     iretd
 892
 893 V86DebugRestore:
 894
 895     /* Get DR0, 1 */
 896     xor ebx, ebx
 897     mov esi, [ebp+KTRAP_FRAME_DR0]
 898     mov edi, [ebp+KTRAP_FRAME_DR1]
 899
 900     /* Clear DR 7 */
 901     mov dr7, ebx
 902
 903     /* Get DR2 and load DR0-2 */
 904     mov ebx, [ebp+KTRAP_FRAME_DR2]
 905     mov dr0, esi
 906     mov dr1, edi
 907     mov dr2, ebx
 908
 909     /* Get DR3-7 */
 910     mov esi, [ebp+KTRAP_FRAME_DR0]
 911     mov edi, [ebp+KTRAP_FRAME_DR1]
 912     mov ebx, [ebp+KTRAP_FRAME_DR7]
 913
 914     /* Load them */
 915     mov dr3, esi
 916     mov dr6, edi
 917     mov dr7, ebx
 918     jmp V86DebugContinue
 919
 920 PendingUserApc:
 921
 922     /* Raise to APC level */
 923     mov ecx, APC_LEVEL
 924     call @KfRaiseIrql@4
 925
 926     /* Save KIRQL and deliver APCs */
 927     push eax
 928     sti
 929     push ebp
 930     push 0
 931     push UserMode
 932     call _KiDeliverApc@12
 933
 934     /* Restore IRQL */
 935     pop ecx
 936     call @KfLowerIrql@4
 937     cli
 938
 939     /* Check if we're not in V86 anymore */
 940     test dword ptr [ebp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
 941     jnz ExitBegin
 942 .endm
 943
 944 //
 945 // @name TRAP_EPILOG
 946 //
 947 // This macro creates an epilogue for leaving any system trap.
 948 // It is used for exiting system calls, exceptions, interrupts and generic
 949 // traps.
 950 //
 951 // @param SystemCall
 952 //        Specifies whether this trap will exit a system call. If so, special
 953 //        code will be assembled to potentially use SYSEXIT instead of IRETD.
 954 //
 955 // @param RestorePreviousMode
 956 //        Specifies if the previous mode should be restored.
 957 //
 958 // @param RestoreSegments
 959 //        Specifies if the segment registers should be restored.
 960 //
 961 // @param RestoreVolatiles
 962 //        Specifies if the volatile registers should be restored.
 963 //
 964 // @param RestoreAllRegs
 965 //        Specifies if volatiles and segments should both be restored.
 966 //
 967 // @remark
 968 //
 969 .macro TRAP_EPILOG SystemCall, RestorePreviousMode, RestoreSegments, RestoreVolatiles, RestoreAllRegs
 970 #ifdef DBG
 971     /* Assert the flags */
 972     pushfd
 973     pop edx
 974     test edx, EFLAGS_INTERRUPT_MASK
 975     jnz 6f
 976
 977     /* Assert the stack */
 978     cmp esp, ebp
 979     jnz 6f
 980
 981     /* Assert the trap frame */
 982 #endif
 983 5:
 984 #ifdef DBG
 985     sub dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
 986     jnz 0f
 987
 988     /* Assert FS */
 989     mov bx, fs
 990     cmp bx, KGDT_R0_PCR
 991     jnz 1f
 992
 993     /* Assert exception list */
 994     cmp dword ptr PCR[KPCR_EXCEPTION_LIST], 0
 995     jnz 2f
 996
 997 1:
 998     push -1
 999     call _KeBugCheck@4
1000 #endif
1001
1002 2:
1003     /* Get exception list */
1004     mov edx, [esp+KTRAP_FRAME_EXCEPTION_LIST]
1005
1006 #ifdef DBG
1007     /* Assert the saved exception list */
1008     or edx, edx
1009     jnz 1f
1010     UNHANDLED_PATH
1011 1:
1012 #endif
1013
1014     /* Restore it */
1015     mov PCR[KPCR_EXCEPTION_LIST], edx
1016
1017 .if \RestorePreviousMode
1018     /* Get previous mode */
1019     mov ecx, [esp+KTRAP_FRAME_PREVIOUS_MODE]
1020
1021 #ifdef DBG
1022     /* Assert the saved previous mode */
1023     cmp ecx, -1
1024     jnz 1f
1025     UNHANDLED_PATH
1026 1:
1027 #endif
1028
1029     /* Restore the previous mode */
1030     mov esi, PCR[KPCR_CURRENT_THREAD]
1031     mov byte ptr [esi+KTHREAD_PREVIOUS_MODE], cl
1032 .else
1033
1034 #ifdef DBG
1035     /* Assert the saved previous mode */
1036     mov ecx, [esp+KTRAP_FRAME_PREVIOUS_MODE]
1037     cmp ecx, -1
1038     jz 1f
1039     UNHANDLED_PATH
1040 1:
1041 #endif
1042 .endif
1043
1044     /* Check for debug registers */
1045     test dword ptr [esp+KTRAP_FRAME_DR7], ~DR7_RESERVED_MASK
1046     jnz 2f
1047
1048     /* Check for V86 */
1049 4:
1050     test dword ptr [esp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
1051     jnz V86_Exit
1052
1053     /* Check if the frame was edited */
1054     test word ptr [esp+KTRAP_FRAME_CS], FRAME_EDITED
1055     jz 7f
1056
1057 .if \RestoreAllRegs
1058     /* Check the old mode */
1059     cmp word ptr [esp+KTRAP_FRAME_CS], KGDT_R3_CODE + RPL_MASK
1060     bt word ptr [esp+KTRAP_FRAME_CS], 0
1061     cmc
1062     ja 8f
1063 .endif
1064
1065 .if \RestoreVolatiles
1066     /* Restore volatiles */
1067     mov edx, [esp+KTRAP_FRAME_EDX]
1068     mov ecx, [esp+KTRAP_FRAME_ECX]
1069     mov eax, [esp+KTRAP_FRAME_EAX]
1070 .endif
1071
1072     /* Check if we were called from kernel mode */
1073     cmp word ptr [ebp+KTRAP_FRAME_CS], KGDT_R0_CODE
1074     jz 9f
1075
1076 .if \RestoreSegments
1077     /* Restore segment registers */
1078     lea esp, [ebp+KTRAP_FRAME_GS]
1079     pop gs
1080     pop es
1081     pop ds
1082 .endif
1083
1084     /* Restore FS */
1085 3:
1086     lea esp, [ebp+KTRAP_FRAME_FS]
1087     pop fs
1088
1089 9:
1090     /* Skip debug information and unsaved registers */
1091     lea esp, [ebp+KTRAP_FRAME_EDI]
1092     pop edi
1093     pop esi
1094     pop ebx
1095     pop ebp
1096
1097     /* Check for ABIOS */
1098     cmp word ptr [esp+8], 0x80
1099     ja AbiosExit
1100
1101     /* Pop error code */
1102     add esp, 4
1103
1104 .if \SystemCall
1105     /* Check if previous CS is from user-mode */
1106     test dword ptr [esp+4], 1
1107
1108     /* It is, so use Fast Exit */
1109     jnz FastExit
1110
1111     /* Jump back to stub */
1112     pop edx
1113     pop ecx
1114     popf
1115     jmp edx
1116
1117 .ret:
1118 .endif
1119     iret
1120
1121 .if \SystemCall
1122 FastExit:
1123     /* Is SYSEXIT Supported/Wanted? */
1124     cmp dword ptr ss:[_KiFastSystemCallDisable], 0
1125     jnz .ret
1126     test dword ptr [esp+8], EFLAGS_TF
1127     jnz .ret
1128
1129     /* Restore FS to TIB */
1130     mov ecx, KGDT_R3_TEB + RPL_MASK
1131     mov fs, ecx
1132
1133     /* We will be cleaning up the stack ourselves */
1134     pop edx                                 /* New Ring 3 EIP */
1135     add esp, 4                              /* Skip Ring 3 DS */
1136     and dword ptr [esp], 0xfffffdff         /* Remove EFLAGS_INTERRUPT_MASK from EFLAGS */
1137     popf                                    /* Restore old EFLAGS */
1138     pop ecx                                 /* Old Ring 3 SS:ESP */
1139
1140     /*
1141      * At this point:
1142      *     ECX points to the old User Stack.
1143      *     EDX points to the instruction to execute in usermode after the sysenter
1144      */
1145     sti
1146     sysexit
1147 .endif
1148
1149 .if \RestoreAllRegs
1150 8:
1151     /* Restore EAX */
1152     mov eax, [esp+KTRAP_FRAME_EAX]
1153
1154     /* Skip registers */
1155     add esp, 0x30
1156
1157     /* Restore segments and volatiles */
1158     pop gs
1159     pop es
1160     pop ds
1161     pop edx
1162     pop ecx
1163
1164     /* Jump back to mainline code */
1165     jmp 3b
1166 .endif
1167
1168 #if DBG
1169 0:
1170     /* Fix up the mask */
1171     add dword ptr [esp+KTRAP_FRAME_DEBUGARGMARK], 0xBADB0D00
1172
1173 6:
1174     UNHANDLED_PATH
1175     jmp 5b
1176 #endif
1177
1178 2:
1179     /* Check if this was V86 mode */
1180     test dword ptr [esp+KTRAP_FRAME_EFLAGS], EFLAGS_V86_MASK
1181     jnz 1f
1182
1183     /* Check if it was user mode */
1184     test word ptr [esp+KTRAP_FRAME_CS], MODE_MASK
1185     jz 4b
1186
1187 1:
1188     /* Clear DR7 */
1189     xor ebx, ebx
1190     mov dr7, ebx
1191
1192     /* Get DR0, 1, 2 */
1193     mov esi, [ebp+KTRAP_FRAME_DR0]
1194     mov edi, [ebp+KTRAP_FRAME_DR1]
1195     mov ebx, [ebp+KTRAP_FRAME_DR2]
1196
1197     /* Set them */
1198     mov dr0, esi
1199     mov dr1, edi
1200     mov dr2, ebx
1201
1202     /* Get DR3, 6, 7 */
1203     mov esi, [ebp+KTRAP_FRAME_DR3]
1204     mov edi, [ebp+KTRAP_FRAME_DR6]
1205     mov ebx, [ebp+KTRAP_FRAME_DR7]
1206
1207     /* Set them */
1208     mov dr3, esi
1209     mov dr6, edi
1210     mov dr7, ebx
1211     jmp 4b
1212
1213 7:
1214     /* Restore real CS value */
1215     mov ebx, [esp+KTRAP_FRAME_TEMPCS]
1216     mov [esp+KTRAP_FRAME_CS], ebx
1217
1218     /*
1219      * If ESP was modified, then a special interrupt exit stack
1220      * must be created to "update" ESP's value in a legal manner
1221      */
1222     mov ebx, [esp+KTRAP_FRAME_TEMPESP]
1223     sub ebx, 0xC
1224     mov [esp+KTRAP_FRAME_ERROR_CODE], ebx
1225
1226     /* Copy Interrupt Stack */
1227     mov esi, [esp+KTRAP_FRAME_EFLAGS]
1228     mov [ebx+8], esi
1229     mov esi, [esp+KTRAP_FRAME_CS]
1230     mov [ebx+4], esi
1231     mov esi, [esp+KTRAP_FRAME_EIP]
1232     mov [ebx], esi
1233
1234 .if \RestoreVolatiles
1235     /* Restore volatiles */
1236     mov eax, [esp+KTRAP_FRAME_EAX]
1237     mov edx, [esp+KTRAP_FRAME_EDX]
1238     mov ecx, [esp+KTRAP_FRAME_ECX]
1239 .endif
1240
1241     /* Return */
1242     add esp, KTRAP_FRAME_EDI
1243     pop edi
1244     pop esi
1245     pop ebx
1246     pop ebp
1247     mov esp, [esp]
1248     iret
1249 .endm
1250
1251 //
1252 // @name INT_EPILOG
1253 //
1254 // This macro creates an epilogue for leaving any system trap.
1255 // It is used for exiting system calls, exceptions, interrupts and generic
1256 // traps.
1257 //
1258 // @param Spurious - TRUE if the interrupt was unexpected and spurious.
1259 //
1260 // @remark None.
1261 //
1262 .macro INT_EPILOG Spurious
1263
1264 .if \Spurious
1265     /* Just exit the trap */
1266     jmp _Kei386EoiHelper@0
1267 .else
1268     /* Disable interrupts */
1269     cli
1270
1271     /* End the interrupt and do EOI */
1272     call _HalEndSystemInterrupt@8
1273     jmp _Kei386EoiHelper@0
1274 .endif
1275 .endm
1276
1277 #ifdef DBG
1278
1279 .macro VERIFY_INT Label
1280     /* Get the current time and mask it to 192 ticks */
1281     mov eax, _KeTickCount
1282     and eax, 0xC0
1283
1284     /* Check if we're in the same tick area */
1285     cmp eax, dword ptr [edi+KINTERRUPT_TICK_COUNT]
1286     jg VfRst_&Label
1287     jl VfWrap_&Label
1288
1289     /* If we got here, then our count is too large */
1290     dec word ptr [edi+KINTERRUPT_DISPATCH_COUNT]
1291     jz VfOvr_&Label
1292 Vf_&Label:
1293 .endm
1294
1295 .macro VERIFY_INT_END Label, Info
1296 VfOvr_&Label:
1297
1298     /* Decrement the dispatch count and check if we should bug check */
1299     dec word ptr [edi+KINTERRUPT_DISPATCH_COUNT+2]
1300     jz 1f
1301
1302     /* Update the tick count */
1303     add eax, 0x40
1304     mov [edi+KINTERRUPT_TICK_COUNT], eax
1305     jmp VfRstDef_&Label
1306
1307 .1:
1308     /* Check if the debugger is enabled */
1309     cmp byte ptr __KdDebuggerEnabled, 0
1310     jnz 1f
1311
1312     /* It isn't, bugcheck */
1313     push Info
1314     push edi
1315     push [edi+KINTERRUPT_SERVICE_CONTEXT]
1316     push [edi+KINTERRUPT_SERVICE_ROUTINE]
1317     push HARDWARE_INTERRUPT_STORM
1318     call _KeBugCheckEx@20
1319
1320 1:
1321     /* Debugger enabled, do a debug print + break instead */
1322     push [edi+KINTERRUPT_SERVICE_ROUTINE]
1323     push offset _IsrOverflowMsg
1324     call _DbgPrint
1325     add esp, 8
1326     int 3
1327
1328     /* Breakpoint handled, get the new tick count */
1329     mov eax, _KeTickCount
1330     and eax, 0xC0
1331
1332 VfRst_&Label:
1333     /* Reset tick count */
1334     mov dword ptr [edi+KINTERRUPT_TICK_COUNT], eax
1335     mov word ptr [edi+KINTERRUPT_DISPATCH_COUNT+2], 64
1336
1337 VfRstDef_&Label:
1338     /* Put default overflow count and continue */
1339     mov ax, _KiISROverflow
1340     mov word ptr [edi+KINTERRUPT_DISPATCH_COUNT], ax
1341     jmp Vf_&Label
1342
1343 VfWrap_&Label:
1344     /* Check if we wrapped */
1345     add eax, 0x40
1346     cmp eax, [edi+KINTERRUPT_TICK_COUNT]
1347     je Vf_&Label
1348
1349     /* We did, start over */
1350     mov eax, _KeTickCount
1351     jmp VfRst_&Label
1352 .endm
1353
1354 #else
1355
1356 /* We don't verify interrupts on retail builds */
1357 .macro VERIFY_INT Label
1358 .endm
1359 .macro VERIFY_INT_END Label, Info
1360 .endm
1361
1362 #endif
1363
1364