ntoskrnl/ke/amd64/trap.S

   1 /*
   2  * FILE:            ntoskrnl/ke/amd64/trap.S
   3  * COPYRIGHT:       See COPYING in the top level directory
   4  * PURPOSE:         System Traps, Entrypoints and Exitpoints
   5  * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
   6  */
   7
   8 /* INCLUDES ******************************************************************/
   9
  10 #include <asm.inc>
  11 #include <ksamd64.inc>
  12 #include <trapamd64.inc>
  13
  14 EXTERN KiDispatchException:PROC
  15 EXTERN FrLdrDbgPrint:DWORD
  16 EXTERN KeBugCheckWithTf:PROC
  17 EXTERN MmAccessFault:PROC
  18 EXTERN KiSystemFatalException:PROC
  19 EXTERN KiNpxNotAvailableFaultHandler:PROC
  20 EXTERN KiGeneralProtectionFaultHandler:PROC
  21 EXTERN KiXmmExceptionHandler:PROC
  22 EXTERN KiDeliverApc:PROC
  23 EXTERN KiDpcInterruptHandler:PROC
  24
  25 /* GLOBALS *******************************************************************/
  26
  27 .data
  28
  29 PUBLIC MsgUnimplemented
  30 MsgUnimplemented:
  31 .asciz "WARNING:  %s at %s:%d is UNIMPLEMENTED!\n"
  32
  33 MsgPageFault:
  34 .asciz "Page fault! Code = 0x%x, RIP = %p, FaultingAddress = %p\n"
  35
  36 MsgGeneralProtFault:
  37 .asciz "General protection fault at %p!\n"
  38
  39 MsgBreakpointTrap:
  40 .asciz "BreakpointTrap at %p\n"
  41
  42 MsgUnexpectedInterrupt:
  43 .asciz "UnexpectedInterrupt Vector=0x%02lx\n"
  44
  45 MsgInvalidOpcodeFault:
  46 .asciz "Invalid opcode fault at %p!\n"
  47
  48 MsgDoubleFault:
  49 .asciz "Double fault at %p, rbp=%p!\n"
  50
  51 MsgTrapInfo:
  52 .asciz "Trap: %s at %p\n"
  53
  54 MACRO(TRAPINFO, func)
  55 LOCAL label1, label2
  56 #if 0
  57     jmp label2
  58 label1:  .asciz "\func"
  59 label2:
  60     push rax
  61     push rcx
  62     push rdx
  63     push r8
  64     push r9
  65     push r10
  66     push r11
  67     sub rsp, 32
  68     lea rcx, MsgTrapInfo[rip]
  69     lea rdx, 1b[rip]
  70     mov r8, [rbp + KTRAP_FRAME_Rip]
  71     call qword ptr FrLdrDbgPrint[rip]
  72     pop r11
  73     pop r10
  74     pop r9
  75     pop r8
  76     pop rdx
  77     pop rcx
  78     pop rax
  79     add rsp, 32
  80 #endif
  81 ENDM
  82
  83 /* Helper Macros *************************************************************/
  84
  85 MACRO(DispatchException, Status, Number, P1, P2, P3)
  86     mov eax, Status
  87     mov edx, Number
  88     mov r9, P1
  89     mov r10, P2
  90     mov r11, P3
  91     call InternalDispatchException
  92 ENDM
  93
  94 MACRO(Fatal, BugcheckCode)
  95     /* Bugcheck */
  96     mov ecx, BugcheckCode
  97     mov rdx, rbp
  98     call KiSystemFatalException
  99 ENDM
 100
 101
 102 /* FUNCTIONS *****************************************************************/
 103
 104 .code64
 105
 106 ALIGN 8
 107
 108 MACRO(UnexpectedVectorStub, Vector)
 109     /* This nop is to make the relative jmp address 4 bytes aligned and to
 110        make the whole code 8 bytes long */
 111     nop
 112     /* This is a push instruction with 8bit operand. Since the instruction
 113        sign extends the value to 32 bits, we need to offset it */
 114 PUBLIC KxUnexpectedInterrupt&Vector
 115 KxUnexpectedInterrupt&Vector:
 116     push (Vector - 128)
 117     jmp KiUnexpectedInterrupt
 118 ENDM
 119
 120 PUBLIC KiUnexpectedRange
 121 KiUnexpectedRange:
 122 Vector = 0
 123 REPEAT 256
 124     UnexpectedVectorStub %Vector
 125     Vector = Vector+1
 126 ENDR
 127 PUBLIC KiUnexpectedRangeEnd
 128 KiUnexpectedRangeEnd:
 129
 130 PUBLIC KiInterruptDispatchTemplate
 131 KiInterruptDispatchTemplate:
 132     /* This instruction pushes the return address on the stack, which is the
 133        address of the interrupt object's DispatchCode member, then jumps
 134        to the address stored in the interrupt object's DispatchAddress member */
 135     call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
 136
 137
 138 // rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
 139 FUNC InternalDispatchException
 140
 141     /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
 142     sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
 143     .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
 144     .endprolog
 145
 146     /* Set up EXCEPTION_RECORD */
 147     lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
 148     mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
 149     xor rax, rax
 150     mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
 151     mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
 152     mov rax, [rbp + KTRAP_FRAME_Rip]
 153     mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
 154     mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
 155     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
 156     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
 157     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
 158
 159     /* Set up KEXCEPTION_FRAME */
 160     mov rax, [rbp + KTRAP_FRAME_Rbp]
 161     mov [rsp + KEXCEPTION_FRAME_Rbp], rax
 162     mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
 163     mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
 164     mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
 165     mov [rsp + KEXCEPTION_FRAME_R12], r12
 166     mov [rsp + KEXCEPTION_FRAME_R13], r13
 167     mov [rsp + KEXCEPTION_FRAME_R14], r14
 168     mov [rsp + KEXCEPTION_FRAME_R15], r15
 169     movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
 170     movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
 171     movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
 172     movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
 173     movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
 174     movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
 175     movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
 176     movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
 177     movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
 178     movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
 179     mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
 180
 181     /* Call KiDispatchException */
 182     // rcx already points to ExceptionRecord
 183     mov rdx, rsp                                // ExceptionFrame
 184     mov r8, rbp                                 // TrapFrame
 185     mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
 186     mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
 187     call KiDispatchException
 188
 189     /* Restore registers */
 190     mov r12, [rsp + KEXCEPTION_FRAME_R12]
 191     mov r13, [rsp + KEXCEPTION_FRAME_R13]
 192     mov r14, [rsp + KEXCEPTION_FRAME_R14]
 193     mov r15, [rsp + KEXCEPTION_FRAME_R15]
 194     movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
 195     movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
 196     movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
 197     movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
 198     movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
 199     movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
 200     movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
 201     movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
 202     movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
 203     movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
 204
 205     add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
 206     ret
 207 ENDFUNC InternalDispatchException
 208
 209
 210 /* CPU EXCEPTION HANDLERS ****************************************************/
 211
 212 PUBLIC KiDivideErrorFault
 213 FUNC KiDivideErrorFault
 214     /* Push pseudo error code */
 215     EnterTrap TF_SAVE_ALL
 216
 217     /* Enable interrupts */
 218     sti
 219
 220     /* Dispatch the exception */
 221     DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
 222
 223     /* Return */
 224     ExitTrap TF_SAVE_ALL
 225 ENDFUNC KiDivideErrorFault
 226
 227
 228 PUBLIC KiDebugTrapOrFault
 229 FUNC KiDebugTrapOrFault
 230     /* Push pseudo error code */
 231     EnterTrap TF_SAVE_ALL
 232
 233     TRAPINFO KiDebugTrapOrFault
 234
 235     /* Check if the frame was from kernelmode */
 236     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 237     jz KiDebugTrapOrFaultKMode
 238
 239     /* Enable interrupts for user-mode */
 240     sti
 241
 242 KiDebugTrapOrFaultKMode:
 243     /* Dispatch the exception */
 244     DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
 245
 246     /* Return */
 247     ExitTrap TF_SAVE_ALL
 248 ENDFUNC KiDebugTrapOrFault
 249
 250
 251 PUBLIC KiNmiInterrupt
 252 FUNC KiNmiInterrupt
 253     /* Push pseudo error code */
 254     EnterTrap TF_SAVE_ALL
 255
 256     UNIMPLEMENTED KiNmiInterrupt
 257     int 3
 258
 259     /* Return */
 260     ExitTrap TF_SAVE_ALL
 261 ENDFUNC KiNmiInterrupt
 262
 263
 264 PUBLIC KiBreakpointTrap
 265 FUNC KiBreakpointTrap
 266     /* Push pseudo error code */
 267     EnterTrap TF_SAVE_ALL
 268
 269     /* Check if the frame was from kernelmode */
 270     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 271     jz KiBreakpointTrapKMode
 272
 273     /* Enable interrupts for user-mode */
 274     sti
 275
 276 KiBreakpointTrapKMode:
 277     /* Dispatch the exception */
 278     DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
 279
 280     /* Return */
 281     ExitTrap TF_SAVE_ALL
 282 ENDFUNC KiBreakpointTrap
 283
 284
 285 PUBLIC KiOverflowTrap
 286 FUNC KiOverflowTrap
 287     /* Push pseudo error code */
 288     EnterTrap TF_SAVE_ALL
 289
 290     /* Enable interrupts */
 291     sti
 292
 293     /* Dispatch the exception */
 294     DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
 295
 296     /* Return */
 297     ExitTrap TF_SAVE_ALL
 298 ENDFUNC KiOverflowTrap
 299
 300
 301 PUBLIC KiBoundFault
 302 FUNC KiBoundFault
 303    /* No error code */
 304     EnterTrap TF_SAVE_ALL
 305
 306     /* Check if the frame was from kernelmode */
 307     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 308     jnz KiBoundFaultUserMode
 309
 310     /* Bugcheck */
 311     Fatal EXCEPTION_BOUND_CHECK
 312
 313 KiBoundFaultUserMode:
 314     /* Enable interrupts for user-mode */
 315     sti
 316
 317     /* Dispatch the exception */
 318     DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
 319
 320     /* Return */
 321     ExitTrap TF_SAVE_ALL
 322 ENDFUNC KiBoundFault
 323
 324
 325 PUBLIC KiInvalidOpcodeFault
 326 FUNC KiInvalidOpcodeFault
 327    /* No error code */
 328     EnterTrap TF_SAVE_ALL
 329
 330     TRAPINFO KiInvalidOpcodeFault
 331
 332     mov rdx, [rbp + KTRAP_FRAME_Rip]
 333     lea rcx, MsgInvalidOpcodeFault[rip]
 334     call qword ptr FrLdrDbgPrint[rip]
 335
 336     /* Enable interrupts */
 337     sti
 338
 339     /* Check if the frame was from kernelmode */
 340     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 341     jz KiInvalidOpcodeKernel
 342
 343     // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
 344
 345 KiInvalidOpcodeKernel:
 346     /* Kernel mode fault */
 347
 348     /* Dispatch the exception */
 349     DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
 350
 351     /* Return */
 352     ExitTrap TF_SAVE_ALL
 353 ENDFUNC KiInvalidOpcodeFault
 354
 355
 356 PUBLIC KiNpxNotAvailableFault
 357 FUNC KiNpxNotAvailableFault
 358    /* No error code */
 359     EnterTrap TF_SAVE_ALL
 360
 361     /* Call the C handler */
 362     mov rcx, rbp
 363     call KiNpxNotAvailableFaultHandler
 364
 365     /* Check the return status code */
 366     test eax, eax
 367     jz KiNpxNotAvailableFaultExit
 368
 369     /* Dispatch the exception */
 370     DispatchException eax, 3, 0, 0, 0
 371
 372 KiNpxNotAvailableFaultExit:
 373     /* Return */
 374     ExitTrap TF_SAVE_ALL
 375 ENDFUNC KiNpxNotAvailableFault
 376
 377
 378 PUBLIC KiDoubleFaultAbort
 379 FUNC KiDoubleFaultAbort
 380    /* No error code */
 381     EnterTrap TF_SAVE_ALL
 382
 383     lea rcx, MsgDoubleFault[rip]
 384     mov rdx, [rbp + KTRAP_FRAME_FaultAddress]
 385     mov r8, rbp
 386     call qword ptr FrLdrDbgPrint[rip]
 387
 388     /* Bugcheck */
 389     Fatal 8 // EXCEPTION_DOUBLE_FAULT
 390     jmp $
 391 ENDFUNC KiDoubleFaultAbort
 392
 393
 394 PUBLIC KiNpxSegmentOverrunAbort
 395 FUNC KiNpxSegmentOverrunAbort
 396    /* No error code */
 397     EnterTrap TF_SAVE_ALL
 398
 399     /* Bugcheck */
 400     Fatal EXCEPTION_NPX_OVERRUN
 401
 402     jmp $
 403 ENDFUNC KiNpxSegmentOverrunAbort
 404
 405
 406 PUBLIC KiInvalidTssFault
 407 FUNC KiInvalidTssFault
 408     /* We have an error code */
 409     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 410
 411     /* Bugcheck */
 412     Fatal EXCEPTION_INVALID_TSS
 413     jmp $
 414 ENDFUNC KiInvalidTssFault
 415
 416
 417 PUBLIC KiSegmentNotPresentFault
 418 FUNC KiSegmentNotPresentFault
 419     /* We have an error code */
 420     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 421
 422     /* Bugcheck */
 423     Fatal EXCEPTION_SEGMENT_NOT_PRESENT
 424     jmp $
 425 ENDFUNC KiSegmentNotPresentFault
 426
 427
 428 PUBLIC KiStackFault
 429 FUNC KiStackFault
 430     /* We have an error code */
 431     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 432
 433     /* Bugcheck */
 434     Fatal EXCEPTION_STACK_FAULT
 435     jmp $
 436 ENDFUNC KiStackFault
 437
 438
 439 PUBLIC KiGeneralProtectionFault
 440 FUNC KiGeneralProtectionFault
 441     /* We have an error code */
 442     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 443
 444     //TRAPINFO KiGeneralProtectionFault
 445     //mov rdx, [rbp + KTRAP_FRAME_Rip]
 446     //lea rcx, MsgGeneralProtFault[rip]
 447     //call qword ptr FrLdrDbgPrint[rip]
 448
 449     /* Call the C handler */
 450     mov rcx, rbp
 451     call KiGeneralProtectionFaultHandler
 452
 453     /* Check for success */
 454     test eax, eax
 455     jge KiGpfExit
 456
 457     /* Dispatch the exception */
 458     DispatchException eax, 3, 0, 0, 0
 459
 460 KiGpfFatal:
 461
 462     /* Bugcheck */
 463     mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
 464     mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
 465     xor r8, r8
 466     mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
 467     sub rsp, 8
 468     mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
 469     call KeBugCheckWithTf
 470
 471 KiGpfExit:
 472     /* Return */
 473     /* Return */
 474     ExitTrap TF_SAVE_ALL
 475 ENDFUNC KiGeneralProtectionFault
 476
 477
 478 PUBLIC KiPageFault
 479 FUNC KiPageFault
 480     /* We have an error code */
 481     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 482
 483     TRAPINFO KiPageFault
 484
 485 #if 0
 486     lea rcx, MsgPageFault[rip]
 487     mov rdx, [rbp + KTRAP_FRAME_ErrorCode]
 488     mov r8, [rbp + KTRAP_FRAME_Rip]
 489     mov r9, [rbp + KTRAP_FRAME_FaultAddress]
 490     call qword ptr FrLdrDbgPrint[rip]
 491 #endif
 492
 493     /* Save page fault address */
 494     mov rdx, cr2
 495     mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
 496
 497     /* Enable interrupts for the page fault handler */
 498     sti
 499
 500     /* Call page fault handler */
 501     mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // StoreInstruction
 502     and ecx, 1
 503     // rdx == Address
 504     mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
 505     and r8b, 1
 506     mov r9, rbp // TrapInformation
 507     call MmAccessFault
 508
 509     /* Check for success */
 510     test eax, eax
 511     jge PageFaultReturn
 512
 513     /* Disable interrupts again for the debugger */
 514     cli
 515
 516     /* Set parameter 1 to error code */
 517     mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
 518
 519     /* Set parameter2 to faulting address */
 520     mov r10, cr2  // Param2 = faulting address
 521
 522     cmp eax, STATUS_ACCESS_VIOLATION
 523     je AccessViolation
 524     cmp eax, STATUS_GUARD_PAGE_VIOLATION
 525     je SpecialCode
 526     cmp eax, STATUS_STACK_OVERFLOW
 527     je SpecialCode
 528
 529 InPageException:
 530
 531     /* Dispatch in-page exception */
 532     mov r11d, eax // Param3 = Status
 533     mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
 534     mov edx, 3    // ParamCount
 535     call InternalDispatchException
 536     jmp PageFaultReturn
 537
 538 AccessViolation:
 539     /* Use more proper status code */
 540     mov eax, KI_EXCEPTION_ACCESS_VIOLATION
 541
 542 SpecialCode:
 543     /* Setup a normal page fault exception */
 544     mov edx, 2   // ParamCount
 545     call InternalDispatchException
 546
 547 PageFaultReturn:
 548     /* Return */
 549     ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
 550 ENDFUNC
 551
 552
 553 PUBLIC KiFloatingErrorFault
 554 FUNC KiFloatingErrorFault
 555    /* No error code */
 556     EnterTrap TF_SAVE_ALL
 557
 558     UNIMPLEMENTED KiFloatingErrorFault
 559     int 3
 560
 561     /* Return */
 562     ExitTrap TF_SAVE_ALL
 563 ENDFUNC KiFloatingErrorFault
 564
 565
 566 PUBLIC KiAlignmentFault
 567 FUNC KiAlignmentFault
 568     /* We have an error code */
 569     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 570
 571     /* Bugcheck */
 572     Fatal EXCEPTION_ALIGNMENT_CHECK
 573     jmp $
 574 ENDFUNC KiAlignmentFault
 575
 576
 577 PUBLIC KiMcheckAbort
 578 FUNC KiMcheckAbort
 579    /* No error code */
 580     EnterTrap TF_SAVE_ALL
 581
 582     /* Bugcheck */
 583     Fatal HEX(12)
 584     jmp $
 585 ENDFUNC KiMcheckAbort
 586
 587
 588 PUBLIC KiXmmException
 589 FUNC KiXmmException
 590    /* No error code */
 591     EnterTrap TF_SAVE_ALL
 592
 593     /* Call the C handler */
 594     mov rcx, rbp
 595     call KiXmmExceptionHandler
 596
 597     /* Check for success */
 598     test eax, eax
 599     jge KiXmmExit
 600
 601     /* Dispatch the exception */
 602     DispatchException eax, 3, 0, 0, 0
 603
 604 KiXmmExit:
 605     /* Return */
 606     ExitTrap TF_SAVE_ALL
 607 ENDFUNC KiXmmException
 608
 609
 610 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
 611
 612 PUBLIC KiRaiseAssertion
 613 FUNC KiRaiseAssertion
 614     /* We have an error code */
 615     EnterTrap (TF_SAVE_ALL)
 616
 617     /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
 618     sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
 619
 620     /* Dispatch the exception */
 621     DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
 622
 623     /* Return */
 624     ExitTrap TF_SAVE_ALL
 625 ENDFUNC KiRaiseAssertion
 626
 627
 628 PUBLIC KiDebugServiceTrap
 629 FUNC KiDebugServiceTrap
 630    /* No error code */
 631     EnterTrap TF_SAVE_ALL
 632
 633     TRAPINFO KiDebugServiceTrap
 634
 635     /* Increase Rip to skip the int3 */
 636     inc qword ptr [rbp + KTRAP_FRAME_Rip]
 637
 638     /* Dispatch the exception (Params = service, buffer, legth) */
 639     DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
 640
 641     /* Return */
 642     ExitTrap TF_SAVE_ALL
 643 ENDFUNC KiDebugServiceTrap
 644
 645
 646 PUBLIC KiApcInterrupt
 647 .PROC KiApcInterrupt
 648    /* No error code */
 649     EnterTrap (TF_VOLATILES or TF_IRQL)
 650
 651     /* Raise to APC_LEVEL */
 652     mov rax, APC_LEVEL
 653     mov cr8, rax
 654
 655     /* End the interrupt */
 656     mov dword ptr [APIC_EOI], 0
 657
 658     /* Enable interrupts  */
 659     sti
 660
 661     /* Call the worker routine */
 662     mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
 663     and cl, 1
 664     mov rdx, 0                        // ExceptionFrame
 665     mov r8, rdx                       // TrapFrame
 666     call KiDeliverApc
 667
 668     /* Disable interrupts */
 669     cli
 670
 671     /* Lower IRQL back to PASSIVE */
 672     mov rax, PASSIVE_LEVEL
 673     mov cr8, rax
 674
 675     /* Return */
 676     ExitTrap (TF_VOLATILES or TF_IRQL)
 677 .ENDP
 678
 679 EXTERN KiRetireDpcList:PROC
 680 PUBLIC KiRetireDpcListInDpcStack
 681 .PROC KiRetireDpcListInDpcStack
 682     push rbp
 683     .pushreg rbp
 684     mov rbp, rsp
 685     .setframe rbp, 0
 686     .endprolog
 687
 688     /* Switch stack and call the function */
 689     mov rsp, rdx
 690     sub rsp, 40
 691     call KiRetireDpcList
 692
 693     /* Restore stack, cleanup and return */
 694     mov rsp, rbp
 695     pop rbp
 696     ret
 697 .ENDP
 698
 699 PUBLIC KiDpcInterrupt
 700 .PROC KiDpcInterrupt
 701    /* No error code */
 702     EnterTrap (TF_VOLATILES or TF_IRQL)
 703
 704     /* Call the worker routine */
 705     call KiDpcInterruptHandler
 706
 707     /* Return, but don't send an EOI! */
 708     ExitTrap (TF_VOLATILES or TF_IRQL)
 709 .ENDP
 710
 711
 712 PUBLIC KiIpiInterrupt
 713 .PROC KiIpiInterrupt
 714    /* No error code */
 715     EnterTrap (TF_VOLATILES or TF_IRQL)
 716
 717     /* Raise to IPI_LEVEL */
 718     mov rax, IPI_LEVEL
 719     mov cr8, rax
 720
 721     /* End the interrupt */
 722     mov dword ptr [APIC_EOI], 0
 723
 724     int 3
 725
 726     /* Return */
 727     ExitTrap (TF_VOLATILES or TF_IRQL)
 728 .ENDP
 729
 730
 731 PUBLIC KiUnexpectedInterrupt
 732 FUNC KiUnexpectedInterrupt
 733     /* The error code is the vector */
 734     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 735
 736 #if 0
 737     /* Set bugcheck parameters */
 738     mov ecx, TRAP_CAUSE_UNKNOWN
 739     mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
 740     mov r8, 0 // The unknown floating-point exception
 741     mov r9, 0 // The enabled and asserted status bits
 742     sub rsp, 8
 743     mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
 744     call KeBugCheckWithTf
 745     jmp $
 746 #endif
 747     /* Return */
 748     ExitTrap TF_SAVE_ALL
 749 ENDFUNC KiUnexpectedInterrupt
 750
 751 PUBLIC KiInterruptDispatch
 752 FUNC KiInterruptDispatch
 753     /* The error code is a pointer to the interrupt object's code */
 754     EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
 755
 756     /* Increase interrupt count */
 757     inc dword ptr gs:[PcInterruptCount];
 758
 759     /* Load the address of the interrupt object into rcx */
 760     mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
 761
 762     /* Substract offset of the DispatchCode member plus 6 for the call instruction */
 763     sub rcx, KINTERRUPT_DispatchCode + 6
 764
 765     /* Raise IRQL to SynchronizeIrql */
 766     movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
 767     mov cr8, rax
 768
 769 #ifdef CONFIG_SMP
 770     /* Acquire interrupt lock */
 771     mov r8, [rcx + KINTERRUPT_ActualLock]
 772
 773     //KxAcquireSpinLock(Interrupt->ActualLock);
 774 #endif
 775
 776     /* Call the ISR */
 777     mov rdx, [rcx + KINTERRUPT_ServiceContext]
 778     call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
 779
 780 #ifdef CONFIG_SMP
 781     /* Release interrupt lock */
 782     //KxReleaseSpinLock(Interrupt->ActualLock);
 783 #endif
 784
 785     /* Go back to old irql */
 786     movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
 787     mov cr8, rax
 788
 789     /* Return */
 790     ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
 791 ENDFUNC
 792
 793
 794 #define MAX_SYSCALL_PARAM_SIZE (16 * 8)
 795 #define HOME_SIZE 6*8
 796 #define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 797
 798 EXTERN KiSystemCallHandler:PROC
 799
 800 /*! \name KiSystemCallEntry64
 801  *
 802  *  \brief This is the entrypoint for syscalls from 64bit user mode
 803  *
 804  *  \param rax - The system call number
 805  *  \param rcx - User mode return address, set by the syscall instruction
 806  *  \param rdx,r8,r9 - Parameters 2-4 to the service function
 807  *  \param r10 - Parameter 1 to the service function
 808  *  \param r11 - RFLAGS saved by the syscall instruction
 809  *--*/
 810 PUBLIC KiSystemCallEntry64
 811 .PROC KiSystemCallEntry64
 812
 813     /* Old stack pointer is in rcx, lie and say we saved it in rbp */
 814     .setframe rbp, 0
 815     .endprolog
 816
 817     /* Swap gs to kernel, so we can access the PCR */
 818     swapgs
 819
 820     /* Save the user mode rsp in the PCR */
 821     mov gs:[PcUserRsp], rsp
 822
 823     /* Get the kernel stack from the PCR */
 824     mov rsp, gs:[PcRspBase]
 825
 826     /* Allocate a TRAP_FRAME and space for parameters */
 827     sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 828 #if DBG
 829     /* Save rbp and load it with the old stack pointer */
 830     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
 831     mov rbp, gs:[PcUserRsp]
 832 #endif
 833
 834     /* Save important volatiles in the trap frame */
 835     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
 836     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
 837     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
 838     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
 839
 840     /* Set sane segments */
 841     mov ax, (KGDT64_R3_DATA or RPL_MASK)
 842     mov ds, ax
 843     mov es, ax
 844
 845     /* Call the C-handler (will enable interrupts) */
 846     lea rcx, [rsp + SYSCALL_ALLOCATION]
 847     call KiSystemCallHandler
 848
 849     /* Deallocate the handlers home stack frame */
 850     add rsp, HOME_SIZE
 851
 852     /* The return value is the address of the Nt-function */
 853     mov rcx, [rsp + 0]
 854     mov rdx, [rsp + 8]
 855     mov r8, [rsp + 16]
 856     mov r9, [rsp + 24]
 857     call rax
 858
 859 #if DBG
 860     /* Restore rbp */
 861     mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
 862 #endif
 863
 864     /* Disable interrupts for return */
 865     cli
 866
 867     /* Restore old trap frame */
 868     mov rcx, gs:[PcCurrentThread]
 869     mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
 870     mov [rcx + KTHREAD_TrapFrame], rdx
 871
 872     /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
 873     mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
 874     mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
 875
 876     /* Load user mode stack (It was copied to the trap frame) */
 877     mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
 878
 879     /* Swap gs back to user */
 880     swapgs
 881
 882     /* return to user mode */
 883     .byte HEX(48) // REX prefix to return to long mode
 884     sysret
 885 .ENDP
 886
 887
 888 PUBLIC KiSystemCallEntry32
 889 KiSystemCallEntry32:
 890     swapgs
 891     int 3
 892
 893
 894 PUBLIC KiZwSystemService
 895 FUNC KiZwSystemService
 896     push rbp
 897     .pushreg rbp
 898     sub rsp, KTRAP_FRAME_LENGTH
 899     .allocstack KTRAP_FRAME_LENGTH
 900     mov [rsp + KTRAP_FRAME_Rsi], rsi
 901     .savereg rsi, KTRAP_FRAME_Rsi
 902     mov [rsp + KTRAP_FRAME_Rdi], rdi
 903     .savereg rdi, KTRAP_FRAME_Rdi
 904     mov rbp, rsp
 905     .setframe rbp, 0
 906     .endprolog
 907
 908     /* Get current thread */
 909     mov r11, gs:[PcCurrentThread]
 910
 911     /* Save the old trap frame in TrapFrame.Rdx */
 912     mov rdi, [r11 + KTHREAD_TrapFrame]
 913     mov [rbp + KTRAP_FRAME_Rdx], rdi
 914
 915     /* Set the new trap frame and previous mode */
 916     mov [r11 + ThTrapFrame], rbp
 917     mov byte ptr [r11 + KTHREAD_PreviousMode], 0
 918
 919     /* allocate space for parameters */
 920     sub rsp, r10
 921     and rsp, HEX(0fffffffffffffff0)
 922
 923     /* Save rcx */
 924     mov [rbp + KTRAP_FRAME_Rcx], rcx
 925
 926     /* copy parameters to the new location */
 927     lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
 928     lea rdi, [rsp]
 929     mov rcx, r10
 930     shr rcx, 3
 931     rep movsq
 932
 933     /* Restore rcx */
 934     mov rcx, [rbp + KTRAP_FRAME_Rcx]
 935
 936     /* Call the service function */
 937     call rax
 938
 939     /* Restore the old trap frame */
 940     mov r11, gs:[PcCurrentThread]
 941     mov rsi, [rsp + KTRAP_FRAME_Rdx]
 942     mov [r11 + KTHREAD_TrapFrame], rsi
 943
 944     /* Restore rdi and rsi */
 945     mov rsi, [rbp + KTRAP_FRAME_Rsi]
 946     mov rdi, [rbp + KTRAP_FRAME_Rdi]
 947
 948     /* Cleanup the stack and return */
 949     lea rsp, [rbp + KTRAP_FRAME_LENGTH]
 950     pop rbp
 951     ret
 952
 953 ENDFUNC
 954
 955
 956 KiExitToUserApc:
 957     int 3
 958
 959 /*!
 960  * VOID
 961  * DECLSPEC_NORETURN
 962  * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
 963  */
 964 PUBLIC KiServiceExit
 965 KiServiceExit:
 966     mov [rcx + KTRAP_FRAME_Rax], rdx
 967     mov rbp, rcx
 968     mov rsp, rcx
 969
 970     /* Return */
 971     //ExitTrap TF_SAVE_ALL
 972
 973 /*!
 974  * VOID
 975  * DECLSPEC_NORETURN
 976  * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
 977  */
 978 PUBLIC KiServiceExit2
 979 .PROC KiServiceExit2
 980     .ENDPROLOG
 981
 982     mov rbp, rcx
 983     mov rsp, rcx
 984
 985     /* Return */
 986     ExitTrap TF_SAVE_ALL
 987 .ENDP
 988
 989 PUBLIC KiInitializeSegments
 990 KiInitializeSegments:
 991     mov ax, KGDT64_R3_DATA or RPL_MASK
 992     mov gs, ax
 993     swapgs
 994     mov gs, ax
 995     ret
 996
 997
 998 #ifdef _MSC_VER
 999 #undef lgdt
1000 #undef lidt
1001
1002 //void __lgdt(void *Source);
1003 PUBLIC __lgdt
1004 __lgdt:
1005     lgdt fword ptr [rcx]
1006     ret
1007
1008 //void __sgdt(void *Destination);
1009 PUBLIC __sgdt
1010 __sgdt:
1011     sgdt fword ptr [rcx]
1012     ret
1013
1014 // void __lldt(unsigned short Value)
1015 PUBLIC __lldt
1016 __lldt:
1017     lldt cx
1018     ret
1019
1020 //void __sldt(void *Destination);
1021 PUBLIC __sldt
1022 __sldt:
1023     sldt word ptr [rcx]
1024     ret
1025
1026 //void __ltr(unsigned short Source);
1027 PUBLIC __ltr
1028 __ltr:
1029     ltr cx
1030     ret
1031
1032 //void __str(unsigned short *Destination);
1033 PUBLIC __str
1034 __str:
1035     str word ptr [rcx]
1036     ret
1037
1038 PUBLIC __swapgs
1039 __swapgs:
1040     swapgs
1041     ret
1042
1043 #endif
1044
1045 END