ntoskrnl/ke/amd64/trap.S

   1 /*
   2  * FILE:            ntoskrnl/ke/amd64/trap.S
   3  * COPYRIGHT:       See COPYING in the top level directory
   4  * PURPOSE:         System Traps, Entrypoints and Exitpoints
   5  * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
   6  */
   7
   8 /* INCLUDES ******************************************************************/
   9
  10 #include <asm.inc>
  11 #include <ksamd64.inc>
  12 #include <trapamd64.inc>
  13
  14 EXTERN KiDispatchException:PROC
  15 EXTERN FrLdrDbgPrint:DWORD
  16 EXTERN KeBugCheckWithTf:PROC
  17 EXTERN MmAccessFault:PROC
  18 EXTERN KiSystemFatalException:PROC
  19 EXTERN KiNpxNotAvailableFaultHandler:PROC
  20 EXTERN KiGeneralProtectionFaultHandler:PROC
  21 EXTERN KiXmmExceptionHandler:PROC
  22 EXTERN KiDeliverApc:PROC
  23 EXTERN KiDpcInterruptHandler:PROC
  24
  25 #ifdef _WINKD_
  26 EXTERN KdSetOwedBreakpoints:PROC
  27 #endif
  28
  29
  30 /* GLOBALS *******************************************************************/
  31
  32 .data
  33
  34 PUBLIC MsgUnimplemented
  35 MsgUnimplemented:
  36 .asciz "WARNING:  %s at %s:%d is UNIMPLEMENTED!\n"
  37
  38 MsgPageFault:
  39 .asciz "Page fault! Code = 0x%x, RIP = %p, FaultingAddress = %p\n"
  40
  41 MsgGeneralProtFault:
  42 .asciz "General protection fault at %p!\n"
  43
  44 MsgBreakpointTrap:
  45 .asciz "BreakpointTrap at %p\n"
  46
  47 MsgUnexpectedInterrupt:
  48 .asciz "UnexpectedInterrupt Vector=0x%02lx\n"
  49
  50 MsgInvalidOpcodeFault:
  51 .asciz "Invalid opcode fault at %p!\n"
  52
  53 MsgDoubleFault:
  54 .asciz "Double fault at %p, rbp=%p!\n"
  55
  56 MsgTrapInfo:
  57 .asciz "Trap: %s at %p\n"
  58
  59 MACRO(TRAPINFO, func)
  60 LOCAL label1, label2
  61 #if 0
  62     jmp label2
  63 label1:  .asciz "\func"
  64 label2:
  65     push rax
  66     push rcx
  67     push rdx
  68     push r8
  69     push r9
  70     push r10
  71     push r11
  72     sub rsp, 32
  73     lea rcx, MsgTrapInfo[rip]
  74     lea rdx, 1b[rip]
  75     mov r8, [rbp + KTRAP_FRAME_Rip]
  76     call qword ptr FrLdrDbgPrint[rip]
  77     pop r11
  78     pop r10
  79     pop r9
  80     pop r8
  81     pop rdx
  82     pop rcx
  83     pop rax
  84     add rsp, 32
  85 #endif
  86 ENDM
  87
  88 /* Helper Macros *************************************************************/
  89
  90 MACRO(DispatchException, Status, Number, P1, P2, P3)
  91     mov eax, Status
  92     mov edx, Number
  93     mov r9, P1
  94     mov r10, P2
  95     mov r11, P3
  96     call InternalDispatchException
  97 ENDM
  98
  99 MACRO(Fatal, BugcheckCode)
 100     /* Bugcheck */
 101     mov ecx, BugcheckCode
 102     mov rdx, rbp
 103     call KiSystemFatalException
 104 ENDM
 105
 106
 107 /* FUNCTIONS *****************************************************************/
 108
 109 .code64
 110
 111 ALIGN 8
 112
 113 MACRO(UnexpectedVectorStub, Vector)
 114     /* This nop is to make the relative jmp address 4 bytes aligned and to
 115        make the whole code 8 bytes long */
 116     nop
 117     /* This is a push instruction with 8bit operand. Since the instruction
 118        sign extends the value to 32 bits, we need to offset it */
 119 PUBLIC KxUnexpectedInterrupt&Vector
 120 KxUnexpectedInterrupt&Vector:
 121     push (Vector - 128)
 122     jmp KiUnexpectedInterrupt
 123 ENDM
 124
 125 PUBLIC KiUnexpectedRange
 126 KiUnexpectedRange:
 127 Vector = 0
 128 REPEAT 256
 129     UnexpectedVectorStub %Vector
 130     Vector = Vector+1
 131 ENDR
 132 PUBLIC KiUnexpectedRangeEnd
 133 KiUnexpectedRangeEnd:
 134
 135 PUBLIC KiInterruptDispatchTemplate
 136 KiInterruptDispatchTemplate:
 137     /* This instruction pushes the return address on the stack, which is the
 138        address of the interrupt object's DispatchCode member, then jumps
 139        to the address stored in the interrupt object's DispatchAddress member */
 140     call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
 141
 142
 143 // rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
 144 FUNC InternalDispatchException
 145
 146     /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
 147     sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
 148     .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
 149     .endprolog
 150
 151     /* Set up EXCEPTION_RECORD */
 152     lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
 153     mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
 154     xor rax, rax
 155     mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
 156     mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
 157     mov rax, [rbp + KTRAP_FRAME_Rip]
 158     mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
 159     mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
 160     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
 161     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
 162     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
 163
 164     /* Set up KEXCEPTION_FRAME */
 165     mov rax, [rbp + KTRAP_FRAME_Rbp]
 166     mov [rsp + KEXCEPTION_FRAME_Rbp], rax
 167     mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
 168     mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
 169     mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
 170     mov [rsp + KEXCEPTION_FRAME_R12], r12
 171     mov [rsp + KEXCEPTION_FRAME_R13], r13
 172     mov [rsp + KEXCEPTION_FRAME_R14], r14
 173     mov [rsp + KEXCEPTION_FRAME_R15], r15
 174     movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
 175     movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
 176     movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
 177     movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
 178     movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
 179     movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
 180     movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
 181     movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
 182     movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
 183     movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
 184     mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
 185
 186     /* Call KiDispatchException */
 187     // rcx already points to ExceptionRecord
 188     mov rdx, rsp                                // ExceptionFrame
 189     mov r8, rbp                                 // TrapFrame
 190     mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
 191     mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
 192     call KiDispatchException
 193
 194     /* Restore registers */
 195     mov r12, [rsp + KEXCEPTION_FRAME_R12]
 196     mov r13, [rsp + KEXCEPTION_FRAME_R13]
 197     mov r14, [rsp + KEXCEPTION_FRAME_R14]
 198     mov r15, [rsp + KEXCEPTION_FRAME_R15]
 199     movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
 200     movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
 201     movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
 202     movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
 203     movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
 204     movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
 205     movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
 206     movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
 207     movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
 208     movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
 209
 210     add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
 211     ret
 212 ENDFUNC
 213
 214
 215 /* CPU EXCEPTION HANDLERS ****************************************************/
 216
 217 PUBLIC KiDivideErrorFault
 218 FUNC KiDivideErrorFault
 219     /* Push pseudo error code */
 220     EnterTrap TF_SAVE_ALL
 221
 222     /* Enable interrupts */
 223     sti
 224
 225     /* Dispatch the exception */
 226     DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
 227
 228     /* Return */
 229     ExitTrap TF_SAVE_ALL
 230 ENDFUNC
 231
 232
 233 PUBLIC KiDebugTrapOrFault
 234 FUNC KiDebugTrapOrFault
 235     /* Push pseudo error code */
 236     EnterTrap TF_SAVE_ALL
 237
 238     TRAPINFO KiDebugTrapOrFault
 239
 240     /* Check if the frame was from kernelmode */
 241     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 242     jz KiDebugTrapOrFaultKMode
 243
 244     /* Enable interrupts for user-mode */
 245     sti
 246
 247 KiDebugTrapOrFaultKMode:
 248     /* Dispatch the exception */
 249     DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
 250
 251     /* Return */
 252     ExitTrap TF_SAVE_ALL
 253 ENDFUNC
 254
 255
 256 PUBLIC KiNmiInterrupt
 257 FUNC KiNmiInterrupt
 258     /* Push pseudo error code */
 259     EnterTrap TF_SAVE_ALL
 260
 261     UNIMPLEMENTED KiNmiInterrupt
 262     int 3
 263
 264     /* Return */
 265     ExitTrap TF_SAVE_ALL
 266 ENDFUNC
 267
 268
 269 PUBLIC KiBreakpointTrap
 270 FUNC KiBreakpointTrap
 271     /* Push pseudo error code */
 272     EnterTrap TF_SAVE_ALL
 273
 274     /* Check if the frame was from kernelmode */
 275     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 276     jz KiBreakpointTrapKMode
 277
 278     /* Enable interrupts for user-mode */
 279     sti
 280
 281 KiBreakpointTrapKMode:
 282     /* Dispatch the exception */
 283     DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
 284
 285     /* Return */
 286     ExitTrap TF_SAVE_ALL
 287 ENDFUNC
 288
 289
 290 PUBLIC KiOverflowTrap
 291 FUNC KiOverflowTrap
 292     /* Push pseudo error code */
 293     EnterTrap TF_SAVE_ALL
 294
 295     /* Enable interrupts */
 296     sti
 297
 298     /* Dispatch the exception */
 299     DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
 300
 301     /* Return */
 302     ExitTrap TF_SAVE_ALL
 303 ENDFUNC
 304
 305
 306 PUBLIC KiBoundFault
 307 FUNC KiBoundFault
 308    /* No error code */
 309     EnterTrap TF_SAVE_ALL
 310
 311     /* Check if the frame was from kernelmode */
 312     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 313     jnz KiBoundFaultUserMode
 314
 315     /* Bugcheck */
 316     Fatal EXCEPTION_BOUND_CHECK
 317
 318 KiBoundFaultUserMode:
 319     /* Enable interrupts for user-mode */
 320     sti
 321
 322     /* Dispatch the exception */
 323     DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
 324
 325     /* Return */
 326     ExitTrap TF_SAVE_ALL
 327 ENDFUNC
 328
 329
 330 PUBLIC KiInvalidOpcodeFault
 331 FUNC KiInvalidOpcodeFault
 332    /* No error code */
 333     EnterTrap TF_SAVE_ALL
 334
 335     TRAPINFO KiInvalidOpcodeFault
 336
 337     mov rdx, [rbp + KTRAP_FRAME_Rip]
 338     lea rcx, MsgInvalidOpcodeFault[rip]
 339     call qword ptr FrLdrDbgPrint[rip]
 340
 341     /* Enable interrupts */
 342     sti
 343
 344     /* Check if the frame was from kernelmode */
 345     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 346     jz KiInvalidOpcodeKernel
 347
 348     // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
 349
 350 KiInvalidOpcodeKernel:
 351     /* Kernel mode fault */
 352
 353     /* Dispatch the exception */
 354     DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
 355
 356     /* Return */
 357     ExitTrap TF_SAVE_ALL
 358 ENDFUNC
 359
 360
 361 PUBLIC KiNpxNotAvailableFault
 362 FUNC KiNpxNotAvailableFault
 363    /* No error code */
 364     EnterTrap TF_SAVE_ALL
 365
 366     /* Call the C handler */
 367     mov rcx, rbp
 368     call KiNpxNotAvailableFaultHandler
 369
 370     /* Check the return status code */
 371     test eax, eax
 372     jz KiNpxNotAvailableFaultExit
 373
 374     /* Dispatch the exception */
 375     DispatchException eax, 3, 0, 0, 0
 376
 377 KiNpxNotAvailableFaultExit:
 378     /* Return */
 379     ExitTrap TF_SAVE_ALL
 380 ENDFUNC
 381
 382
 383 PUBLIC KiDoubleFaultAbort
 384 FUNC KiDoubleFaultAbort
 385    /* No error code */
 386     EnterTrap TF_SAVE_ALL
 387
 388     lea rcx, MsgDoubleFault[rip]
 389     mov rdx, [rbp + KTRAP_FRAME_FaultAddress]
 390     mov r8, rbp
 391     call qword ptr FrLdrDbgPrint[rip]
 392
 393     /* Bugcheck */
 394     Fatal 8 // EXCEPTION_DOUBLE_FAULT
 395     jmp $
 396 ENDFUNC
 397
 398
 399 PUBLIC KiNpxSegmentOverrunAbort
 400 FUNC KiNpxSegmentOverrunAbort
 401    /* No error code */
 402     EnterTrap TF_SAVE_ALL
 403
 404     /* Bugcheck */
 405     Fatal EXCEPTION_NPX_OVERRUN
 406
 407     jmp $
 408 ENDFUNC
 409
 410
 411 PUBLIC KiInvalidTssFault
 412 FUNC KiInvalidTssFault
 413     /* We have an error code */
 414     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 415
 416     /* Bugcheck */
 417     Fatal EXCEPTION_INVALID_TSS
 418     jmp $
 419 ENDFUNC
 420
 421
 422 PUBLIC KiSegmentNotPresentFault
 423 FUNC KiSegmentNotPresentFault
 424     /* We have an error code */
 425     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 426
 427     /* Bugcheck */
 428     Fatal EXCEPTION_SEGMENT_NOT_PRESENT
 429     jmp $
 430 ENDFUNC
 431
 432
 433 PUBLIC KiStackFault
 434 FUNC KiStackFault
 435     /* We have an error code */
 436     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 437
 438     /* Bugcheck */
 439     Fatal EXCEPTION_STACK_FAULT
 440     jmp $
 441 ENDFUNC
 442
 443
 444 PUBLIC KiGeneralProtectionFault
 445 FUNC KiGeneralProtectionFault
 446     /* We have an error code */
 447     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 448
 449     //TRAPINFO KiGeneralProtectionFault
 450     //mov rdx, [rbp + KTRAP_FRAME_Rip]
 451     //lea rcx, MsgGeneralProtFault[rip]
 452     //call qword ptr FrLdrDbgPrint[rip]
 453
 454     /* Call the C handler */
 455     mov rcx, rbp
 456     call KiGeneralProtectionFaultHandler
 457
 458     /* Check for success */
 459     test eax, eax
 460     jge KiGpfExit
 461
 462     /* Dispatch the exception */
 463     DispatchException eax, 3, 0, 0, 0
 464
 465 KiGpfFatal:
 466
 467     /* Bugcheck */
 468     mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
 469     mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
 470     xor r8, r8
 471     mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
 472     sub rsp, 8
 473     mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
 474     call KeBugCheckWithTf
 475
 476 KiGpfExit:
 477     /* Return */
 478     /* Return */
 479     ExitTrap TF_SAVE_ALL
 480 ENDFUNC
 481
 482
 483 PUBLIC KiPageFault
 484 FUNC KiPageFault
 485     /* We have an error code */
 486     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 487
 488     TRAPINFO KiPageFault
 489
 490 #if 0
 491     lea rcx, MsgPageFault[rip]
 492     mov rdx, [rbp + KTRAP_FRAME_ErrorCode]
 493     mov r8, [rbp + KTRAP_FRAME_Rip]
 494     mov r9, [rbp + KTRAP_FRAME_FaultAddress]
 495     call qword ptr FrLdrDbgPrint[rip]
 496 #endif
 497
 498     /* Save page fault address */
 499     mov rdx, cr2
 500     mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
 501
 502     /* Enable interrupts for the page fault handler */
 503     sti
 504
 505     /* Call page fault handler */
 506     mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // StoreInstruction
 507     and ecx, 1
 508     // rdx == Address
 509     mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
 510     and r8b, 1
 511     mov r9, rbp // TrapInformation
 512     call MmAccessFault
 513
 514     /* Check for success */
 515     test eax, eax
 516 #ifndef _WINKD_
 517     jge PageFaultReturn
 518 #else
 519     jl PageFaultError
 520
 521     /* Check whether the kernel debugger has owed breakpoints to be inserted */
 522     call KdSetOwedBreakpoints
 523     /* We succeeded, return */
 524     jmp PageFaultReturn
 525
 526 PageFaultError:
 527 #endif
 528
 529     /* Disable interrupts again for the debugger */
 530     cli
 531
 532     /* Set parameter 1 to error code */
 533     mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
 534
 535     /* Set parameter2 to faulting address */
 536     mov r10, cr2  // Param2 = faulting address
 537
 538     cmp eax, STATUS_ACCESS_VIOLATION
 539     je AccessViolation
 540     cmp eax, STATUS_GUARD_PAGE_VIOLATION
 541     je SpecialCode
 542     cmp eax, STATUS_STACK_OVERFLOW
 543     je SpecialCode
 544
 545 InPageException:
 546     /* Dispatch in-page exception */
 547     mov r11d, eax // Param3 = Status
 548     mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
 549     mov edx, 3    // ParamCount
 550     call InternalDispatchException
 551     jmp PageFaultReturn
 552
 553 AccessViolation:
 554     /* Use more proper status code */
 555     mov eax, KI_EXCEPTION_ACCESS_VIOLATION
 556
 557 SpecialCode:
 558     /* Setup a normal page fault exception */
 559     mov edx, 2   // ParamCount
 560     call InternalDispatchException
 561
 562 PageFaultReturn:
 563     /* Return */
 564     ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
 565 ENDFUNC
 566
 567
 568 PUBLIC KiFloatingErrorFault
 569 FUNC KiFloatingErrorFault
 570    /* No error code */
 571     EnterTrap TF_SAVE_ALL
 572
 573     UNIMPLEMENTED KiFloatingErrorFault
 574     int 3
 575
 576     /* Return */
 577     ExitTrap TF_SAVE_ALL
 578 ENDFUNC
 579
 580
 581 PUBLIC KiAlignmentFault
 582 FUNC KiAlignmentFault
 583     /* We have an error code */
 584     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 585
 586     /* Bugcheck */
 587     Fatal EXCEPTION_ALIGNMENT_CHECK
 588     jmp $
 589 ENDFUNC
 590
 591
 592 PUBLIC KiMcheckAbort
 593 FUNC KiMcheckAbort
 594    /* No error code */
 595     EnterTrap TF_SAVE_ALL
 596
 597     /* Bugcheck */
 598     Fatal HEX(12)
 599     jmp $
 600 ENDFUNC
 601
 602
 603 PUBLIC KiXmmException
 604 FUNC KiXmmException
 605    /* No error code */
 606     EnterTrap TF_SAVE_ALL
 607
 608     /* Call the C handler */
 609     mov rcx, rbp
 610     call KiXmmExceptionHandler
 611
 612     /* Check for success */
 613     test eax, eax
 614     jge KiXmmExit
 615
 616     /* Dispatch the exception */
 617     DispatchException eax, 3, 0, 0, 0
 618
 619 KiXmmExit:
 620     /* Return */
 621     ExitTrap TF_SAVE_ALL
 622 ENDFUNC
 623
 624
 625 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
 626
 627 PUBLIC KiRaiseAssertion
 628 FUNC KiRaiseAssertion
 629     /* We have an error code */
 630     EnterTrap (TF_SAVE_ALL)
 631
 632     /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
 633     sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
 634
 635     /* Dispatch the exception */
 636     DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
 637
 638     /* Return */
 639     ExitTrap TF_SAVE_ALL
 640 ENDFUNC
 641
 642
 643 PUBLIC KiDebugServiceTrap
 644 FUNC KiDebugServiceTrap
 645    /* No error code */
 646     EnterTrap TF_SAVE_ALL
 647
 648     TRAPINFO KiDebugServiceTrap
 649
 650     /* Increase Rip to skip the int3 */
 651     inc qword ptr [rbp + KTRAP_FRAME_Rip]
 652
 653     /* Dispatch the exception (Params = service, buffer, legth) */
 654     DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
 655
 656     /* Return */
 657     ExitTrap TF_SAVE_ALL
 658 ENDFUNC
 659
 660
 661 PUBLIC KiApcInterrupt
 662 .PROC KiApcInterrupt
 663    /* No error code */
 664     EnterTrap (TF_VOLATILES or TF_IRQL)
 665
 666     /* Raise to APC_LEVEL */
 667     mov rax, APC_LEVEL
 668     mov cr8, rax
 669
 670     /* End the interrupt */
 671     mov dword ptr [APIC_EOI], 0
 672
 673     /* Enable interrupts  */
 674     sti
 675
 676     /* Call the worker routine */
 677     mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
 678     and cl, 1
 679     mov rdx, 0                        // ExceptionFrame
 680     mov r8, rdx                       // TrapFrame
 681     call KiDeliverApc
 682
 683     /* Disable interrupts */
 684     cli
 685
 686     /* Lower IRQL back to PASSIVE */
 687     mov rax, PASSIVE_LEVEL
 688     mov cr8, rax
 689
 690     /* Return */
 691     ExitTrap (TF_VOLATILES or TF_IRQL)
 692 .ENDP
 693
 694 EXTERN KiRetireDpcList:PROC
 695 PUBLIC KiRetireDpcListInDpcStack
 696 .PROC KiRetireDpcListInDpcStack
 697     push rbp
 698     .pushreg rbp
 699     mov rbp, rsp
 700     .setframe rbp, 0
 701     .endprolog
 702
 703     /* Switch stack and call the function */
 704     mov rsp, rdx
 705     sub rsp, 40
 706     call KiRetireDpcList
 707
 708     /* Restore stack, cleanup and return */
 709     mov rsp, rbp
 710     pop rbp
 711     ret
 712 .ENDP
 713
 714 PUBLIC KiDpcInterrupt
 715 .PROC KiDpcInterrupt
 716    /* No error code */
 717     EnterTrap (TF_VOLATILES or TF_IRQL)
 718
 719     /* Call the worker routine */
 720     call KiDpcInterruptHandler
 721
 722     /* Return, but don't send an EOI! */
 723     ExitTrap (TF_VOLATILES or TF_IRQL)
 724 .ENDP
 725
 726
 727 PUBLIC KiIpiInterrupt
 728 .PROC KiIpiInterrupt
 729    /* No error code */
 730     EnterTrap (TF_VOLATILES or TF_IRQL)
 731
 732     /* Raise to IPI_LEVEL */
 733     mov rax, IPI_LEVEL
 734     mov cr8, rax
 735
 736     /* End the interrupt */
 737     mov dword ptr [APIC_EOI], 0
 738
 739     int 3
 740
 741     /* Return */
 742     ExitTrap (TF_VOLATILES or TF_IRQL)
 743 .ENDP
 744
 745
 746 PUBLIC KiUnexpectedInterrupt
 747 FUNC KiUnexpectedInterrupt
 748     /* The error code is the vector */
 749     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 750
 751 #if 0
 752     /* Set bugcheck parameters */
 753     mov ecx, TRAP_CAUSE_UNKNOWN
 754     mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
 755     mov r8, 0 // The unknown floating-point exception
 756     mov r9, 0 // The enabled and asserted status bits
 757     sub rsp, 8
 758     mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
 759     call KeBugCheckWithTf
 760     jmp $
 761 #endif
 762     /* Return */
 763     ExitTrap TF_SAVE_ALL
 764 ENDFUNC
 765
 766 PUBLIC KiInterruptDispatch
 767 FUNC KiInterruptDispatch
 768     /* The error code is a pointer to the interrupt object's code */
 769     EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
 770
 771     /* Increase interrupt count */
 772     inc dword ptr gs:[PcInterruptCount];
 773
 774     /* Load the address of the interrupt object into rcx */
 775     mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
 776
 777     /* Substract offset of the DispatchCode member plus 6 for the call instruction */
 778     sub rcx, KINTERRUPT_DispatchCode + 6
 779
 780     /* Raise IRQL to SynchronizeIrql */
 781     movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
 782     mov cr8, rax
 783
 784 #ifdef CONFIG_SMP
 785     /* Acquire interrupt lock */
 786     mov r8, [rcx + KINTERRUPT_ActualLock]
 787
 788     //KxAcquireSpinLock(Interrupt->ActualLock);
 789 #endif
 790
 791     /* Call the ISR */
 792     mov rdx, [rcx + KINTERRUPT_ServiceContext]
 793     call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
 794
 795 #ifdef CONFIG_SMP
 796     /* Release interrupt lock */
 797     //KxReleaseSpinLock(Interrupt->ActualLock);
 798 #endif
 799
 800     /* Go back to old irql */
 801     movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
 802     mov cr8, rax
 803
 804     /* Return */
 805     ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
 806 ENDFUNC
 807
 808
 809 #define MAX_SYSCALL_PARAM_SIZE (16 * 8)
 810 #define HOME_SIZE 6*8
 811 #define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 812
 813 EXTERN KiSystemCallHandler:PROC
 814
 815 /*! \name KiSystemCallEntry64
 816  *
 817  *  \brief This is the entrypoint for syscalls from 64bit user mode
 818  *
 819  *  \param rax - The system call number
 820  *  \param rcx - User mode return address, set by the syscall instruction
 821  *  \param rdx,r8,r9 - Parameters 2-4 to the service function
 822  *  \param r10 - Parameter 1 to the service function
 823  *  \param r11 - RFLAGS saved by the syscall instruction
 824  *--*/
 825 PUBLIC KiSystemCallEntry64
 826 .PROC KiSystemCallEntry64
 827
 828     /* Old stack pointer is in rcx, lie and say we saved it in rbp */
 829     .setframe rbp, 0
 830     .endprolog
 831
 832     /* Swap gs to kernel, so we can access the PCR */
 833     swapgs
 834
 835     /* Save the user mode rsp in the PCR */
 836     mov gs:[PcUserRsp], rsp
 837
 838     /* Get the kernel stack from the PCR */
 839     mov rsp, gs:[PcRspBase]
 840
 841     /* Allocate a TRAP_FRAME and space for parameters */
 842     sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 843 #if DBG
 844     /* Save rbp and load it with the old stack pointer */
 845     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
 846     mov rbp, gs:[PcUserRsp]
 847 #endif
 848
 849     /* Save important volatiles in the trap frame */
 850     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
 851     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
 852     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
 853     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
 854
 855     /* Set sane segments */
 856     mov ax, (KGDT64_R3_DATA or RPL_MASK)
 857     mov ds, ax
 858     mov es, ax
 859
 860     /* Call the C-handler (will enable interrupts) */
 861     lea rcx, [rsp + SYSCALL_ALLOCATION]
 862     call KiSystemCallHandler
 863
 864     /* Deallocate the handlers home stack frame */
 865     add rsp, HOME_SIZE
 866
 867     /* The return value is the address of the Nt-function */
 868     mov rcx, [rsp + 0]
 869     mov rdx, [rsp + 8]
 870     mov r8, [rsp + 16]
 871     mov r9, [rsp + 24]
 872     call rax
 873
 874 #if DBG
 875     /* Restore rbp */
 876     mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
 877 #endif
 878
 879     /* Disable interrupts for return */
 880     cli
 881
 882     /* Restore old trap frame */
 883     mov rcx, gs:[PcCurrentThread]
 884     mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
 885     mov [rcx + KTHREAD_TrapFrame], rdx
 886
 887     /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
 888     mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
 889     mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
 890
 891     /* Load user mode stack (It was copied to the trap frame) */
 892     mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
 893
 894     /* Swap gs back to user */
 895     swapgs
 896
 897     /* return to user mode */
 898     .byte HEX(48) // REX prefix to return to long mode
 899     sysret
 900 .ENDP
 901
 902
 903 PUBLIC KiSystemCallEntry32
 904 KiSystemCallEntry32:
 905     swapgs
 906     int 3
 907
 908
 909 PUBLIC KiZwSystemService
 910 FUNC KiZwSystemService
 911     push rbp
 912     .pushreg rbp
 913     sub rsp, KTRAP_FRAME_LENGTH
 914     .allocstack KTRAP_FRAME_LENGTH
 915     mov [rsp + KTRAP_FRAME_Rsi], rsi
 916     .savereg rsi, KTRAP_FRAME_Rsi
 917     mov [rsp + KTRAP_FRAME_Rdi], rdi
 918     .savereg rdi, KTRAP_FRAME_Rdi
 919     mov rbp, rsp
 920     .setframe rbp, 0
 921     .endprolog
 922
 923     /* Get current thread */
 924     mov r11, gs:[PcCurrentThread]
 925
 926     /* Save the old trap frame in TrapFrame.Rdx */
 927     mov rdi, [r11 + KTHREAD_TrapFrame]
 928     mov [rbp + KTRAP_FRAME_Rdx], rdi
 929
 930     /* Set the new trap frame and previous mode */
 931     mov [r11 + ThTrapFrame], rbp
 932     mov byte ptr [r11 + KTHREAD_PreviousMode], 0
 933
 934     /* allocate space for parameters */
 935     sub rsp, r10
 936     and rsp, HEX(0fffffffffffffff0)
 937
 938     /* Save rcx */
 939     mov [rbp + KTRAP_FRAME_Rcx], rcx
 940
 941     /* copy parameters to the new location */
 942     lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
 943     lea rdi, [rsp]
 944     mov rcx, r10
 945     shr rcx, 3
 946     rep movsq
 947
 948     /* Restore rcx */
 949     mov rcx, [rbp + KTRAP_FRAME_Rcx]
 950
 951     /* Call the service function */
 952     call rax
 953
 954     /* Restore the old trap frame */
 955     mov r11, gs:[PcCurrentThread]
 956     mov rsi, [rsp + KTRAP_FRAME_Rdx]
 957     mov [r11 + KTHREAD_TrapFrame], rsi
 958
 959     /* Restore rdi and rsi */
 960     mov rsi, [rbp + KTRAP_FRAME_Rsi]
 961     mov rdi, [rbp + KTRAP_FRAME_Rdi]
 962
 963     /* Cleanup the stack and return */
 964     lea rsp, [rbp + KTRAP_FRAME_LENGTH]
 965     pop rbp
 966     ret
 967
 968 ENDFUNC
 969
 970
 971 KiExitToUserApc:
 972     int 3
 973
 974 /*!
 975  * VOID
 976  * DECLSPEC_NORETURN
 977  * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
 978  */
 979 PUBLIC KiServiceExit
 980 KiServiceExit:
 981     mov [rcx + KTRAP_FRAME_Rax], rdx
 982     mov rbp, rcx
 983     mov rsp, rcx
 984
 985     /* Return */
 986     //ExitTrap TF_SAVE_ALL
 987
 988 /*!
 989  * VOID
 990  * DECLSPEC_NORETURN
 991  * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
 992  */
 993 PUBLIC KiServiceExit2
 994 .PROC KiServiceExit2
 995     .ENDPROLOG
 996
 997     mov rbp, rcx
 998     mov rsp, rcx
 999
1000     /* Return */
1001     ExitTrap TF_SAVE_ALL
1002 .ENDP
1003
1004 PUBLIC KiInitializeSegments
1005 KiInitializeSegments:
1006     mov ax, KGDT64_R3_DATA or RPL_MASK
1007     mov gs, ax
1008     swapgs
1009     mov gs, ax
1010     ret
1011
1012
1013 #ifdef _MSC_VER
1014 #undef lgdt
1015 #undef lidt
1016
1017 //void __lgdt(void *Source);
1018 PUBLIC __lgdt
1019 __lgdt:
1020     lgdt fword ptr [rcx]
1021     ret
1022
1023 //void __sgdt(void *Destination);
1024 PUBLIC __sgdt
1025 __sgdt:
1026     sgdt fword ptr [rcx]
1027     ret
1028
1029 // void __lldt(unsigned short Value)
1030 PUBLIC __lldt
1031 __lldt:
1032     lldt cx
1033     ret
1034
1035 //void __sldt(void *Destination);
1036 PUBLIC __sldt
1037 __sldt:
1038     sldt word ptr [rcx]
1039     ret
1040
1041 //void __ltr(unsigned short Source);
1042 PUBLIC __ltr
1043 __ltr:
1044     ltr cx
1045     ret
1046
1047 //void __str(unsigned short *Destination);
1048 PUBLIC __str
1049 __str:
1050     str word ptr [rcx]
1051     ret
1052
1053 PUBLIC __swapgs
1054 __swapgs:
1055     swapgs
1056     ret
1057
1058 #endif
1059
1060 END