ntoskrnl/ke/amd64/trap.S

   1 /*
   2  * FILE:            ntoskrnl/ke/amd64/trap.S
   3  * COPYRIGHT:       See COPYING in the top level directory
   4  * PURPOSE:         System Traps, Entrypoints and Exitpoints
   5  * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
   6  */
   7
   8 /* INCLUDES ******************************************************************/
   9
  10 #include <asm.inc>
  11 #include <ksamd64.inc>
  12 #include <trapamd64.inc>
  13
  14 EXTERN KiDispatchException:PROC
  15 EXTERN FrLdrDbgPrint:DWORD
  16 EXTERN KeBugCheckWithTf:PROC
  17 EXTERN MmAccessFault:PROC
  18 EXTERN KiSystemFatalException:PROC
  19 EXTERN KiNpxNotAvailableFaultHandler:PROC
  20 EXTERN KiGeneralProtectionFaultHandler:PROC
  21 EXTERN KiXmmExceptionHandler:PROC
  22 EXTERN KiDeliverApc:PROC
  23 EXTERN KiDpcInterruptHandler:PROC
  24
  25 /* GLOBALS *******************************************************************/
  26
  27 .data
  28
  29 PUBLIC MsgUnimplemented
  30 MsgUnimplemented:
  31 .asciz "WARNING:  %s at %s:%d is UNIMPLEMENTED!\n"
  32
  33
  34 /* Helper Macros *************************************************************/
  35
  36 MACRO(DispatchException, Status, Number, P1, P2, P3)
  37     mov eax, Status
  38     mov edx, Number
  39     mov r9, P1
  40     mov r10, P2
  41     mov r11, P3
  42     call InternalDispatchException
  43 ENDM
  44
  45 MACRO(Fatal, BugcheckCode)
  46     /* Bugcheck */
  47     mov ecx, BugcheckCode
  48     mov rdx, rbp
  49     call KiSystemFatalException
  50 ENDM
  51
  52
  53 /* FUNCTIONS *****************************************************************/
  54
  55 .code64
  56
  57 ALIGN 8
  58
  59 MACRO(UnexpectedVectorStub, Vector)
  60     /* This nop is to make the relative jmp address 4 bytes aligned and to
  61        make the whole code 8 bytes long */
  62     nop
  63     /* This is a push instruction with 8bit operand. Since the instruction
  64        sign extends the value to 32 bits, we need to offset it */
  65 PUBLIC KxUnexpectedInterrupt&Vector
  66 KxUnexpectedInterrupt&Vector:
  67     push (Vector - 128)
  68     jmp KiUnexpectedInterrupt
  69 ENDM
  70
  71 PUBLIC KiUnexpectedRange
  72 KiUnexpectedRange:
  73 Vector = 0
  74 REPEAT 256
  75     UnexpectedVectorStub %Vector
  76     Vector = Vector+1
  77 ENDR
  78 PUBLIC KiUnexpectedRangeEnd
  79 KiUnexpectedRangeEnd:
  80
  81 PUBLIC KiInterruptDispatchTemplate
  82 KiInterruptDispatchTemplate:
  83     /* This instruction pushes the return address on the stack, which is the
  84        address of the interrupt object's DispatchCode member, then jumps
  85        to the address stored in the interrupt object's DispatchAddress member */
  86     call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
  87
  88
  89 // rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
  90 .PROC InternalDispatchException
  91
  92     /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
  93     sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
  94     .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
  95     .endprolog
  96
  97     /* Set up EXCEPTION_RECORD */
  98     lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
  99     mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
 100     xor rax, rax
 101     mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
 102     mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
 103     mov rax, [rbp + KTRAP_FRAME_Rip]
 104     mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
 105     mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
 106     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
 107     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
 108     mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
 109
 110     /* Set up KEXCEPTION_FRAME */
 111     mov rax, [rbp + KTRAP_FRAME_Rbp]
 112     mov [rsp + KEXCEPTION_FRAME_Rbp], rax
 113     mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
 114     mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
 115     mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
 116     mov [rsp + KEXCEPTION_FRAME_R12], r12
 117     mov [rsp + KEXCEPTION_FRAME_R13], r13
 118     mov [rsp + KEXCEPTION_FRAME_R14], r14
 119     mov [rsp + KEXCEPTION_FRAME_R15], r15
 120     movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
 121     movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
 122     movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
 123     movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
 124     movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
 125     movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
 126     movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
 127     movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
 128     movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
 129     movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
 130     mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
 131
 132     /* Call KiDispatchException */
 133     // rcx already points to ExceptionRecord
 134     mov rdx, rsp                                // ExceptionFrame
 135     mov r8, rbp                                 // TrapFrame
 136     mov r9b, [r8 + KTRAP_FRAME_PreviousMode]    // PreviousMode
 137     mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
 138     call KiDispatchException
 139
 140     /* Restore registers */
 141     mov r12, [rsp + KEXCEPTION_FRAME_R12]
 142     mov r13, [rsp + KEXCEPTION_FRAME_R13]
 143     mov r14, [rsp + KEXCEPTION_FRAME_R14]
 144     mov r15, [rsp + KEXCEPTION_FRAME_R15]
 145     movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
 146     movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
 147     movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
 148     movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
 149     movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
 150     movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
 151     movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
 152     movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
 153     movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
 154     movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
 155
 156     add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
 157     ret
 158 .ENDP
 159
 160
 161 /* CPU EXCEPTION HANDLERS ****************************************************/
 162
 163 PUBLIC KiDivideErrorFault
 164 FUNC KiDivideErrorFault
 165     /* Push pseudo error code */
 166     EnterTrap TF_SAVE_ALL
 167
 168     /* Enable interrupts */
 169     sti
 170
 171     /* Dispatch the exception */
 172     DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
 173
 174     /* Return */
 175     ExitTrap TF_SAVE_ALL
 176 ENDFUNC
 177
 178
 179 PUBLIC KiDebugTrapOrFault
 180 FUNC KiDebugTrapOrFault
 181     /* Push pseudo error code */
 182     EnterTrap TF_SAVE_ALL
 183
 184     /* Check if the frame was from kernelmode */
 185     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 186     jz KiDebugTrapOrFaultKMode
 187
 188     /* Enable interrupts for user-mode */
 189     sti
 190
 191 KiDebugTrapOrFaultKMode:
 192     /* Dispatch the exception */
 193     DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
 194
 195     /* Return */
 196     ExitTrap TF_SAVE_ALL
 197 ENDFUNC
 198
 199
 200 PUBLIC KiNmiInterrupt
 201 FUNC KiNmiInterrupt
 202     /* Push pseudo error code */
 203     EnterTrap TF_SAVE_ALL
 204
 205     UNIMPLEMENTED KiNmiInterrupt
 206     int 3
 207
 208     /* Return */
 209     ExitTrap TF_SAVE_ALL
 210 ENDFUNC
 211
 212
 213 PUBLIC KiBreakpointTrap
 214 FUNC KiBreakpointTrap
 215     /* Push pseudo error code */
 216     EnterTrap TF_SAVE_ALL
 217
 218     /* Check if the frame was from kernelmode */
 219     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 220     jz KiBreakpointTrapKMode
 221
 222     /* Enable interrupts for user-mode */
 223     sti
 224
 225 KiBreakpointTrapKMode:
 226     /* Dispatch the exception */
 227     DispatchException STATUS_BREAKPOINT, 3, 0, 0, 0
 228
 229     /* Return */
 230     ExitTrap TF_SAVE_ALL
 231 ENDFUNC
 232
 233
 234 PUBLIC KiOverflowTrap
 235 FUNC KiOverflowTrap
 236     /* Push pseudo error code */
 237     EnterTrap TF_SAVE_ALL
 238
 239     /* Enable interrupts */
 240     sti
 241
 242     /* Dispatch the exception */
 243     DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
 244
 245     /* Return */
 246     ExitTrap TF_SAVE_ALL
 247 ENDFUNC
 248
 249
 250 PUBLIC KiBoundFault
 251 FUNC KiBoundFault
 252    /* No error code */
 253     EnterTrap TF_SAVE_ALL
 254
 255     /* Check if the frame was from kernelmode */
 256     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 257     jnz KiBoundFaultUserMode
 258
 259     /* Bugcheck */
 260     Fatal EXCEPTION_BOUND_CHECK
 261
 262 KiBoundFaultUserMode:
 263     /* Enable interrupts for user-mode */
 264     sti
 265
 266     /* Dispatch the exception */
 267     DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
 268
 269     /* Return */
 270     ExitTrap TF_SAVE_ALL
 271 ENDFUNC
 272
 273
 274 PUBLIC KiInvalidOpcodeFault
 275 FUNC KiInvalidOpcodeFault
 276    /* No error code */
 277     EnterTrap TF_SAVE_ALL
 278
 279     /* Enable interrupts */
 280     sti
 281
 282     /* Check if the frame was from kernelmode */
 283     test word ptr [rbp + KTRAP_FRAME_SegCs], 3
 284     jz KiInvalidOpcodeKernel
 285
 286     // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
 287
 288 KiInvalidOpcodeKernel:
 289     /* Kernel mode fault */
 290
 291     /* Dispatch the exception */
 292     DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
 293
 294     /* Return */
 295     ExitTrap TF_SAVE_ALL
 296 ENDFUNC
 297
 298
 299 PUBLIC KiNpxNotAvailableFault
 300 FUNC KiNpxNotAvailableFault
 301    /* No error code */
 302     EnterTrap TF_SAVE_ALL
 303
 304     /* Call the C handler */
 305     mov rcx, rbp
 306     call KiNpxNotAvailableFaultHandler
 307
 308     /* Check the return status code */
 309     test eax, eax
 310     jz KiNpxNotAvailableFaultExit
 311
 312     /* Dispatch the exception */
 313     DispatchException eax, 3, 0, 0, 0
 314
 315 KiNpxNotAvailableFaultExit:
 316     /* Return */
 317     ExitTrap TF_SAVE_ALL
 318 ENDFUNC
 319
 320
 321 PUBLIC KiDoubleFaultAbort
 322 FUNC KiDoubleFaultAbort
 323    /* No error code */
 324     EnterTrap TF_SAVE_ALL
 325
 326     /* Bugcheck */
 327     Fatal 8 // EXCEPTION_DOUBLE_FAULT
 328     jmp $
 329 ENDFUNC
 330
 331
 332 PUBLIC KiNpxSegmentOverrunAbort
 333 FUNC KiNpxSegmentOverrunAbort
 334    /* No error code */
 335     EnterTrap TF_SAVE_ALL
 336
 337     /* Bugcheck */
 338     Fatal EXCEPTION_NPX_OVERRUN
 339     jmp $
 340 ENDFUNC
 341
 342
 343 PUBLIC KiInvalidTssFault
 344 FUNC KiInvalidTssFault
 345     /* We have an error code */
 346     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 347
 348     /* Bugcheck */
 349     Fatal EXCEPTION_INVALID_TSS
 350     jmp $
 351 ENDFUNC
 352
 353
 354 PUBLIC KiSegmentNotPresentFault
 355 FUNC KiSegmentNotPresentFault
 356     /* We have an error code */
 357     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 358
 359     /* Bugcheck */
 360     Fatal EXCEPTION_SEGMENT_NOT_PRESENT
 361     jmp $
 362 ENDFUNC
 363
 364
 365 PUBLIC KiStackFault
 366 FUNC KiStackFault
 367     /* We have an error code */
 368     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 369
 370     /* Bugcheck */
 371     Fatal EXCEPTION_STACK_FAULT
 372     jmp $
 373 ENDFUNC
 374
 375
 376 PUBLIC KiGeneralProtectionFault
 377 FUNC KiGeneralProtectionFault
 378     /* We have an error code */
 379     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 380
 381     /* Call the C handler */
 382     mov rcx, rbp
 383     call KiGeneralProtectionFaultHandler
 384
 385     /* Check for success */
 386     test eax, eax
 387     jge KiGpfExit
 388
 389     /* Dispatch the exception */
 390     DispatchException eax, 3, 0, 0, 0
 391
 392 KiGpfFatal:
 393
 394     /* Bugcheck */
 395     mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
 396     mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
 397     xor r8, r8
 398     mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
 399     sub rsp, 8
 400     mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
 401     call KeBugCheckWithTf
 402
 403 KiGpfExit:
 404     /* Return */
 405     /* Return */
 406     ExitTrap TF_SAVE_ALL
 407 ENDFUNC
 408
 409
 410 PUBLIC KiPageFault
 411 FUNC KiPageFault
 412     /* We have an error code */
 413     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 414
 415     /* Save page fault address */
 416     mov rdx, cr2
 417     mov [rbp  + KTRAP_FRAME_FaultAddress], rdx
 418
 419     /* Enable interrupts for the page fault handler */
 420     sti
 421
 422     /* Call page fault handler */
 423     mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // StoreInstruction
 424     and ecx, 1
 425     // rdx == Address
 426     mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
 427     and r8b, 1
 428     mov r9, rbp // TrapInformation
 429     call MmAccessFault
 430
 431     /* Check for success */
 432     test eax, eax
 433     jge PageFaultReturn
 434
 435     /* Disable interrupts again for the debugger */
 436     cli
 437
 438     /* Set parameter 1 to error code */
 439     mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
 440
 441     /* Set parameter2 to faulting address */
 442     mov r10, cr2  // Param2 = faulting address
 443
 444     cmp eax, STATUS_ACCESS_VIOLATION
 445     je AccessViolation
 446     cmp eax, STATUS_GUARD_PAGE_VIOLATION
 447     je SpecialCode
 448     cmp eax, STATUS_STACK_OVERFLOW
 449     je SpecialCode
 450
 451 InPageException:
 452
 453     /* Dispatch in-page exception */
 454     mov r11d, eax // Param3 = Status
 455     mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
 456     mov edx, 3    // ParamCount
 457     call InternalDispatchException
 458     jmp PageFaultReturn
 459
 460 AccessViolation:
 461     /* Use more proper status code */
 462     mov eax, KI_EXCEPTION_ACCESS_VIOLATION
 463
 464 SpecialCode:
 465     /* Setup a normal page fault exception */
 466     mov edx, 2   // ParamCount
 467     call InternalDispatchException
 468
 469 PageFaultReturn:
 470     /* Return */
 471     ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
 472 ENDFUNC
 473
 474
 475 PUBLIC KiFloatingErrorFault
 476 FUNC KiFloatingErrorFault
 477    /* No error code */
 478     EnterTrap TF_SAVE_ALL
 479
 480     UNIMPLEMENTED KiFloatingErrorFault
 481     int 3
 482
 483     /* Return */
 484     ExitTrap TF_SAVE_ALL
 485 ENDFUNC
 486
 487
 488 PUBLIC KiAlignmentFault
 489 FUNC KiAlignmentFault
 490     /* We have an error code */
 491     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 492
 493     /* Bugcheck */
 494     Fatal EXCEPTION_ALIGNMENT_CHECK
 495     jmp $
 496 ENDFUNC
 497
 498
 499 PUBLIC KiMcheckAbort
 500 FUNC KiMcheckAbort
 501    /* No error code */
 502     EnterTrap TF_SAVE_ALL
 503
 504     /* Bugcheck */
 505     Fatal HEX(12)
 506     jmp $
 507 ENDFUNC
 508
 509
 510 PUBLIC KiXmmException
 511 FUNC KiXmmException
 512    /* No error code */
 513     EnterTrap TF_SAVE_ALL
 514
 515     /* Call the C handler */
 516     mov rcx, rbp
 517     call KiXmmExceptionHandler
 518
 519     /* Check for success */
 520     test eax, eax
 521     jge KiXmmExit
 522
 523     /* Dispatch the exception */
 524     DispatchException eax, 3, 0, 0, 0
 525
 526 KiXmmExit:
 527     /* Return */
 528     ExitTrap TF_SAVE_ALL
 529 ENDFUNC
 530
 531
 532 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
 533
 534 PUBLIC KiRaiseAssertion
 535 FUNC KiRaiseAssertion
 536     /* We have an error code */
 537     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 538
 539     /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
 540     sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
 541
 542     /* Dispatch the exception */
 543     DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
 544
 545     /* Return */
 546     ExitTrap TF_SAVE_ALL
 547 ENDFUNC
 548
 549
 550 PUBLIC KiDebugServiceTrap
 551 .PROC KiDebugServiceTrap
 552    /* No error code */
 553     EnterTrap TF_SAVE_ALL
 554
 555     /* Increase Rip to skip the int3 */
 556     inc qword ptr [rbp + KTRAP_FRAME_Rip]
 557
 558     /* Dispatch the exception (Params = service, buffer, legth) */
 559     DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
 560
 561     /* Return */
 562     ExitTrap TF_SAVE_ALL
 563 .ENDP
 564
 565
 566 PUBLIC KiApcInterrupt
 567 .PROC KiApcInterrupt
 568    /* No error code */
 569     EnterTrap (TF_VOLATILES or TF_IRQL)
 570
 571     /* Raise to APC_LEVEL */
 572     mov rax, APC_LEVEL
 573     mov cr8, rax
 574
 575     /* End the interrupt */
 576     mov dword ptr [APIC_EOI], 0
 577
 578     /* Enable interrupts  */
 579     sti
 580
 581     /* Call the worker routine */
 582     mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
 583     and cl, 1
 584     mov rdx, 0                        // ExceptionFrame
 585     mov r8, rdx                       // TrapFrame
 586     call KiDeliverApc
 587
 588     /* Disable interrupts */
 589     cli
 590
 591     /* Lower IRQL back to PASSIVE */
 592     mov rax, PASSIVE_LEVEL
 593     mov cr8, rax
 594
 595     /* Return */
 596     ExitTrap (TF_VOLATILES or TF_IRQL)
 597 .ENDP
 598
 599 EXTERN KiRetireDpcList:PROC
 600 PUBLIC KiRetireDpcListInDpcStack
 601 .PROC KiRetireDpcListInDpcStack
 602     push rbp
 603     .pushreg rbp
 604     mov rbp, rsp
 605     .setframe rbp, 0
 606     .endprolog
 607
 608     /* Switch stack and call the function */
 609     mov rsp, rdx
 610     sub rsp, 40
 611     call KiRetireDpcList
 612
 613     /* Restore stack, cleanup and return */
 614     mov rsp, rbp
 615     pop rbp
 616     ret
 617 .ENDP
 618
 619 PUBLIC KiDpcInterrupt
 620 .PROC KiDpcInterrupt
 621    /* No error code */
 622     EnterTrap (TF_VOLATILES or TF_IRQL)
 623
 624     /* Call the worker routine */
 625     call KiDpcInterruptHandler
 626
 627     /* Return, but don't send an EOI! */
 628     ExitTrap (TF_VOLATILES or TF_IRQL)
 629 .ENDP
 630
 631
 632 PUBLIC KiIpiInterrupt
 633 .PROC KiIpiInterrupt
 634    /* No error code */
 635     EnterTrap (TF_VOLATILES or TF_IRQL)
 636
 637     /* Raise to IPI_LEVEL */
 638     mov rax, IPI_LEVEL
 639     mov cr8, rax
 640
 641     /* End the interrupt */
 642     mov dword ptr [APIC_EOI], 0
 643
 644     int 3
 645
 646     /* Return */
 647     ExitTrap (TF_VOLATILES or TF_IRQL)
 648 .ENDP
 649
 650
 651 PUBLIC KiUnexpectedInterrupt
 652 FUNC KiUnexpectedInterrupt
 653     /* The error code is the vector */
 654     EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
 655
 656 #if 0
 657     /* Set bugcheck parameters */
 658     mov ecx, TRAP_CAUSE_UNKNOWN
 659     mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
 660     mov r8, 0 // The unknown floating-point exception
 661     mov r9, 0 // The enabled and asserted status bits
 662     sub rsp, 8
 663     mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
 664     call KeBugCheckWithTf
 665     jmp $
 666 #endif
 667     /* Return */
 668     ExitTrap TF_SAVE_ALL
 669 ENDFUNC
 670
 671 PUBLIC KiInterruptDispatch
 672 FUNC KiInterruptDispatch
 673     /* The error code is a pointer to the interrupt object's code */
 674     EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
 675
 676     /* Increase interrupt count */
 677     inc dword ptr gs:[PcInterruptCount];
 678
 679     /* Load the address of the interrupt object into rcx */
 680     mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
 681
 682     /* Substract offset of the DispatchCode member plus 6 for the call instruction */
 683     sub rcx, KINTERRUPT_DispatchCode + 6
 684
 685     /* Raise IRQL to SynchronizeIrql */
 686     movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
 687     mov cr8, rax
 688
 689 #ifdef CONFIG_SMP
 690     /* Acquire interrupt lock */
 691     mov r8, [rcx + KINTERRUPT_ActualLock]
 692
 693     //KxAcquireSpinLock(Interrupt->ActualLock);
 694 #endif
 695
 696     /* Call the ISR */
 697     mov rdx, [rcx + KINTERRUPT_ServiceContext]
 698     call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
 699
 700 #ifdef CONFIG_SMP
 701     /* Release interrupt lock */
 702     //KxReleaseSpinLock(Interrupt->ActualLock);
 703 #endif
 704
 705     /* Go back to old irql */
 706     movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
 707     mov cr8, rax
 708
 709     /* Return */
 710     ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
 711 ENDFUNC
 712
 713
 714 #define MAX_SYSCALL_PARAM_SIZE (16 * 8)
 715 #define HOME_SIZE 6*8
 716 #define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 717
 718 EXTERN KiSystemCallHandler:PROC
 719
 720 /*! \name KiSystemCallEntry64
 721  *
 722  *  \brief This is the entrypoint for syscalls from 64bit user mode
 723  *
 724  *  \param rax - The system call number
 725  *  \param rcx - User mode return address, set by the syscall instruction
 726  *  \param rdx,r8,r9 - Parameters 2-4 to the service function
 727  *  \param r10 - Parameter 1 to the service function
 728  *  \param r11 - RFLAGS saved by the syscall instruction
 729  *--*/
 730 PUBLIC KiSystemCallEntry64
 731 .PROC KiSystemCallEntry64
 732
 733     /* Old stack pointer is in rcx, lie and say we saved it in rbp */
 734     .setframe rbp, 0
 735     .endprolog
 736
 737     /* Swap gs to kernel, so we can access the PCR */
 738     swapgs
 739
 740     /* Save the user mode rsp in the PCR */
 741     mov gs:[PcUserRsp], rsp
 742
 743     /* Get the kernel stack from the PCR */
 744     mov rsp, gs:[PcRspBase]
 745
 746     /* Allocate a TRAP_FRAME and space for parameters */
 747     sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
 748 #if DBG
 749     /* Save rbp and load it with the old stack pointer */
 750     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
 751     mov rbp, gs:[PcUserRsp]
 752 #endif
 753
 754     /* Save important volatiles in the trap frame */
 755     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
 756     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
 757     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
 758     mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
 759
 760     /* Set sane segments */
 761     mov ax, (KGDT64_R3_DATA or RPL_MASK)
 762     mov ds, ax
 763     mov es, ax
 764
 765     /* Call the C-handler (will enable interrupts) */
 766     lea rcx, [rsp + SYSCALL_ALLOCATION]
 767     call KiSystemCallHandler
 768
 769     /* Deallocate the handlers home stack frame */
 770     add rsp, HOME_SIZE
 771
 772     /* The return value is the address of the Nt-function */
 773     mov rcx, [rsp + 0]
 774     mov rdx, [rsp + 8]
 775     mov r8, [rsp + 16]
 776     mov r9, [rsp + 24]
 777     call rax
 778
 779 #if DBG
 780     /* Restore rbp */
 781     mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
 782 #endif
 783
 784     /* Disable interrupts for return */
 785     cli
 786
 787     /* Restore old trap frame */
 788     mov rcx, gs:[PcCurrentThread]
 789     mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
 790     mov [rcx + KTHREAD_TrapFrame], rdx
 791
 792     /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
 793     mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
 794     mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
 795
 796     /* Load user mode stack (It was copied to the trap frame) */
 797     mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
 798
 799     /* Swap gs back to user */
 800     swapgs
 801
 802     /* return to user mode */
 803     .byte HEX(48) // REX prefix to return to long mode
 804     sysret
 805 .ENDP
 806
 807
 808 PUBLIC KiSystemCallEntry32
 809 KiSystemCallEntry32:
 810     swapgs
 811     int 3
 812
 813
 814 PUBLIC KiZwSystemService
 815 FUNC KiZwSystemService
 816     push rbp
 817     .pushreg rbp
 818     sub rsp, KTRAP_FRAME_LENGTH
 819     .allocstack KTRAP_FRAME_LENGTH
 820     mov [rsp + KTRAP_FRAME_Rsi], rsi
 821     .savereg rsi, KTRAP_FRAME_Rsi
 822     mov [rsp + KTRAP_FRAME_Rdi], rdi
 823     .savereg rdi, KTRAP_FRAME_Rdi
 824     mov rbp, rsp
 825     .setframe rbp, 0
 826     .endprolog
 827
 828     /* Get current thread */
 829     mov r11, gs:[PcCurrentThread]
 830
 831     /* Save the old trap frame in TrapFrame.Rdx */
 832     mov rdi, [r11 + KTHREAD_TrapFrame]
 833     mov [rbp + KTRAP_FRAME_Rdx], rdi
 834
 835     /* Set the new trap frame and previous mode */
 836     mov [r11 + ThTrapFrame], rbp
 837     mov byte ptr [r11 + KTHREAD_PreviousMode], 0
 838
 839     /* allocate space for parameters */
 840     sub rsp, r10
 841     and rsp, HEX(0fffffffffffffff0)
 842
 843     /* Save rcx */
 844     mov [rbp + KTRAP_FRAME_Rcx], rcx
 845
 846     /* copy parameters to the new location */
 847     lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
 848     lea rdi, [rsp]
 849     mov rcx, r10
 850     shr rcx, 3
 851     rep movsq
 852
 853     /* Restore rcx */
 854     mov rcx, [rbp + KTRAP_FRAME_Rcx]
 855
 856     /* Call the service function */
 857     call rax
 858
 859     /* Restore the old trap frame */
 860     mov r11, gs:[PcCurrentThread]
 861     mov rsi, [rsp + KTRAP_FRAME_Rdx]
 862     mov [r11 + KTHREAD_TrapFrame], rsi
 863
 864     /* Restore rdi and rsi */
 865     mov rsi, [rbp + KTRAP_FRAME_Rsi]
 866     mov rdi, [rbp + KTRAP_FRAME_Rdi]
 867
 868     /* Cleanup the stack and return */
 869     lea rsp, [rbp + KTRAP_FRAME_LENGTH]
 870     pop rbp
 871     ret
 872
 873 ENDFUNC
 874
 875
 876 KiExitToUserApc:
 877     int 3
 878
 879 /*!
 880  * VOID
 881  * DECLSPEC_NORETURN
 882  * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
 883  */
 884 PUBLIC KiServiceExit
 885 KiServiceExit:
 886     mov [rcx + KTRAP_FRAME_Rax], rdx
 887     mov rbp, rcx
 888     mov rsp, rcx
 889
 890     /* Return */
 891     //ExitTrap TF_SAVE_ALL
 892
 893 /*!
 894  * VOID
 895  * DECLSPEC_NORETURN
 896  * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
 897  */
 898 PUBLIC KiServiceExit2
 899 .PROC KiServiceExit2
 900     .ENDPROLOG
 901
 902     mov rbp, rcx
 903     mov rsp, rcx
 904
 905     /* Return */
 906     ExitTrap TF_SAVE_ALL
 907 .ENDP
 908
 909 PUBLIC KiInitializeSegments
 910 KiInitializeSegments:
 911     mov ax, KGDT64_R3_DATA or RPL_MASK
 912     mov gs, ax
 913     swapgs
 914     mov gs, ax
 915     ret
 916
 917
 918 #ifdef _MSC_VER
 919 #undef lgdt
 920 #undef lidt
 921
 922 //void __lgdt(void *Source);
 923 PUBLIC __lgdt
 924 __lgdt:
 925     lgdt fword ptr [rcx]
 926     ret
 927
 928 //void __sgdt(void *Destination);
 929 PUBLIC __sgdt
 930 __sgdt:
 931     sgdt fword ptr [rcx]
 932     ret
 933
 934 // void __lldt(unsigned short Value)
 935 PUBLIC __lldt
 936 __lldt:
 937     lldt cx
 938     ret
 939
 940 //void __sldt(void *Destination);
 941 PUBLIC __sldt
 942 __sldt:
 943     sldt word ptr [rcx]
 944     ret
 945
 946 //void __ltr(unsigned short Source);
 947 PUBLIC __ltr
 948 __ltr:
 949     ltr cx
 950     ret
 951
 952 //void __str(unsigned short *Destination);
 953 PUBLIC __str
 954 __str:
 955     str word ptr [rcx]
 956     ret
 957
 958 PUBLIC __swapgs
 959 __swapgs:
 960     swapgs
 961     ret
 962
 963 #endif
 964
 965 END