[NTOS:KE] Fix pushing of fake error code in KiDoubleFaultAbort (amd64)
[reactos.git] / ntoskrnl / ke / amd64 / trap.S
1 /*
2 * FILE: ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT: See COPYING in the top level directory
4 * PURPOSE: System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER: Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8 /* INCLUDES ******************************************************************/
9
10 #include <asm.inc>
11 #include <ksamd64.inc>
12 #include <trapamd64.inc>
13
14 EXTERN KiDispatchException:PROC
15 EXTERN FrLdrDbgPrint:DWORD
16 EXTERN KeBugCheckWithTf:PROC
17 EXTERN MmAccessFault:PROC
18 EXTERN KiSystemFatalException:PROC
19 EXTERN KiNpxNotAvailableFaultHandler:PROC
20 EXTERN KiGeneralProtectionFaultHandler:PROC
21 EXTERN KiXmmExceptionHandler:PROC
22 EXTERN KiDeliverApc:PROC
23 EXTERN KiDpcInterruptHandler:PROC
24
25 #ifdef _WINKD_
26 EXTERN KdSetOwedBreakpoints:PROC
27 #endif
28
29
30 /* GLOBALS *******************************************************************/
31
32 .data
33
34 PUBLIC MsgUnimplemented
35 MsgUnimplemented:
36 .asciz "WARNING: %s at %s:%d is UNIMPLEMENTED!\n"
37
38 MsgPageFault:
39 .asciz "Page fault! Code = 0x%x, RIP = %p, FaultingAddress = %p\n"
40
41 MsgGeneralProtFault:
42 .asciz "General protection fault at %p!\n"
43
44 MsgBreakpointTrap:
45 .asciz "BreakpointTrap at %p\n"
46
47 MsgUnexpectedInterrupt:
48 .asciz "UnexpectedInterrupt Vector=0x%02lx\n"
49
50 MsgInvalidOpcodeFault:
51 .asciz "Invalid opcode fault at %p!\n"
52
53 MsgDoubleFault:
54 .asciz "Double fault at %p, rbp=%p!\n"
55
56 MsgTrapInfo:
57 .asciz "Trap: %s at %p\n"
58
59 MACRO(TRAPINFO, func)
60 LOCAL label1, label2
61 #if 0
62 jmp label2
63 label1: .asciz "\func"
64 label2:
65 push rax
66 push rcx
67 push rdx
68 push r8
69 push r9
70 push r10
71 push r11
72 sub rsp, 32
73 lea rcx, MsgTrapInfo[rip]
74 lea rdx, 1b[rip]
75 mov r8, [rbp + KTRAP_FRAME_Rip]
76 call qword ptr FrLdrDbgPrint[rip]
77 pop r11
78 pop r10
79 pop r9
80 pop r8
81 pop rdx
82 pop rcx
83 pop rax
84 add rsp, 32
85 #endif
86 ENDM
87
88 /* Helper Macros *************************************************************/
89
90 MACRO(DispatchException, Status, Number, P1, P2, P3)
91 mov eax, Status
92 mov edx, Number
93 mov r9, P1
94 mov r10, P2
95 mov r11, P3
96 call InternalDispatchException
97 ENDM
98
99 MACRO(Fatal, BugcheckCode)
100 /* Bugcheck */
101 mov ecx, BugcheckCode
102 mov rdx, rbp
103 call KiSystemFatalException
104 ENDM
105
106
107 /* FUNCTIONS *****************************************************************/
108
109 .code64
110
111 ALIGN 8
112
113 MACRO(UnexpectedVectorStub, Vector)
114 /* This nop is to make the relative jmp address 4 bytes aligned and to
115 make the whole code 8 bytes long */
116 nop
117 /* This is a push instruction with 8bit operand. Since the instruction
118 sign extends the value to 32 bits, we need to offset it */
119 PUBLIC KxUnexpectedInterrupt&Vector
120 KxUnexpectedInterrupt&Vector:
121 push (Vector - 128)
122 jmp KiUnexpectedInterrupt
123 ENDM
124
125 PUBLIC KiUnexpectedRange
126 KiUnexpectedRange:
127 Vector = 0
128 REPEAT 256
129 UnexpectedVectorStub %Vector
130 Vector = Vector+1
131 ENDR
132 PUBLIC KiUnexpectedRangeEnd
133 KiUnexpectedRangeEnd:
134
135 PUBLIC KiInterruptDispatchTemplate
136 KiInterruptDispatchTemplate:
137 /* This instruction pushes the return address on the stack, which is the
138 address of the interrupt object's DispatchCode member, then jumps
139 to the address stored in the interrupt object's DispatchAddress member */
140 call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
141
142
143 // rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
144 FUNC InternalDispatchException
145
146 /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
147 sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
148 .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
149 .endprolog
150
151 /* Set up EXCEPTION_RECORD */
152 lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
153 mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
154 xor rax, rax
155 mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
156 mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
157 mov rax, [rbp + KTRAP_FRAME_Rip]
158 mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
159 mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
160 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
161 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
162 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
163
164 /* Set up KEXCEPTION_FRAME */
165 mov rax, [rbp + KTRAP_FRAME_Rbp]
166 mov [rsp + KEXCEPTION_FRAME_Rbp], rax
167 mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
168 mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
169 mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
170 mov [rsp + KEXCEPTION_FRAME_R12], r12
171 mov [rsp + KEXCEPTION_FRAME_R13], r13
172 mov [rsp + KEXCEPTION_FRAME_R14], r14
173 mov [rsp + KEXCEPTION_FRAME_R15], r15
174 movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
175 movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
176 movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
177 movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
178 movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
179 movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
180 movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
181 movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
182 movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
183 movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
184 mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
185
186 /* Call KiDispatchException */
187 // rcx already points to ExceptionRecord
188 mov rdx, rsp // ExceptionFrame
189 mov r8, rbp // TrapFrame
190 mov r9b, [r8 + KTRAP_FRAME_PreviousMode] // PreviousMode
191 mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
192 call KiDispatchException
193
194 /* Restore registers */
195 mov r12, [rsp + KEXCEPTION_FRAME_R12]
196 mov r13, [rsp + KEXCEPTION_FRAME_R13]
197 mov r14, [rsp + KEXCEPTION_FRAME_R14]
198 mov r15, [rsp + KEXCEPTION_FRAME_R15]
199 movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
200 movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
201 movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
202 movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
203 movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
204 movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
205 movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
206 movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
207 movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
208 movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
209
210 add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
211 ret
212 ENDFUNC
213
214
215 /* CPU EXCEPTION HANDLERS ****************************************************/
216
217 PUBLIC KiDivideErrorFault
218 FUNC KiDivideErrorFault
219 /* Push pseudo error code */
220 EnterTrap TF_SAVE_ALL
221
222 /* Enable interrupts */
223 sti
224
225 /* Dispatch the exception */
226 DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
227
228 /* Return */
229 ExitTrap TF_SAVE_ALL
230 ENDFUNC
231
232
233 PUBLIC KiDebugTrapOrFault
234 FUNC KiDebugTrapOrFault
235 /* Push pseudo error code */
236 EnterTrap TF_SAVE_ALL
237
238 TRAPINFO KiDebugTrapOrFault
239
240 /* Check if the frame was from kernelmode */
241 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
242 jz KiDebugTrapOrFaultKMode
243
244 /* Enable interrupts for user-mode */
245 sti
246
247 KiDebugTrapOrFaultKMode:
248 /* Dispatch the exception */
249 DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
250
251 /* Return */
252 ExitTrap TF_SAVE_ALL
253 ENDFUNC
254
255
256 PUBLIC KiNmiInterrupt
257 FUNC KiNmiInterrupt
258 /* Push pseudo error code */
259 EnterTrap TF_SAVE_ALL
260
261 UNIMPLEMENTED KiNmiInterrupt
262 int 3
263
264 /* Return */
265 ExitTrap TF_SAVE_ALL
266 ENDFUNC
267
268
269 PUBLIC KiBreakpointTrap
270 FUNC KiBreakpointTrap
271 /* Push pseudo error code */
272 EnterTrap TF_SAVE_ALL
273
274 /* Check if the frame was from kernelmode */
275 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
276 jz KiBreakpointTrapKMode
277
278 /* Enable interrupts for user-mode */
279 sti
280
281 KiBreakpointTrapKMode:
282 /* Dispatch the exception */
283 DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
284
285 /* Return */
286 ExitTrap TF_SAVE_ALL
287 ENDFUNC
288
289
290 PUBLIC KiOverflowTrap
291 FUNC KiOverflowTrap
292 /* Push pseudo error code */
293 EnterTrap TF_SAVE_ALL
294
295 /* Enable interrupts */
296 sti
297
298 /* Dispatch the exception */
299 DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
300
301 /* Return */
302 ExitTrap TF_SAVE_ALL
303 ENDFUNC
304
305
306 PUBLIC KiBoundFault
307 FUNC KiBoundFault
308 /* Push pseudo error code */
309 EnterTrap TF_SAVE_ALL
310
311 /* Check if the frame was from kernelmode */
312 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
313 jnz KiBoundFaultUserMode
314
315 /* Bugcheck */
316 Fatal EXCEPTION_BOUND_CHECK
317
318 KiBoundFaultUserMode:
319 /* Enable interrupts for user-mode */
320 sti
321
322 /* Dispatch the exception */
323 DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
324
325 /* Return */
326 ExitTrap TF_SAVE_ALL
327 ENDFUNC
328
329
330 PUBLIC KiInvalidOpcodeFault
331 FUNC KiInvalidOpcodeFault
332 /* Push pseudo error code */
333 EnterTrap TF_SAVE_ALL
334
335 TRAPINFO KiInvalidOpcodeFault
336
337 mov rdx, [rbp + KTRAP_FRAME_Rip]
338 lea rcx, MsgInvalidOpcodeFault[rip]
339 call qword ptr FrLdrDbgPrint[rip]
340
341 /* Enable interrupts */
342 sti
343
344 /* Check if the frame was from kernelmode */
345 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
346 jz KiInvalidOpcodeKernel
347
348 // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
349
350 KiInvalidOpcodeKernel:
351 /* Kernel mode fault */
352
353 /* Dispatch the exception */
354 DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
355
356 /* Return */
357 ExitTrap TF_SAVE_ALL
358 ENDFUNC
359
360
361 PUBLIC KiNpxNotAvailableFault
362 FUNC KiNpxNotAvailableFault
363 /* Push pseudo error code */
364 EnterTrap TF_SAVE_ALL
365
366 /* Call the C handler */
367 mov rcx, rbp
368 call KiNpxNotAvailableFaultHandler
369
370 /* Check the return status code */
371 test eax, eax
372 jz KiNpxNotAvailableFaultExit
373
374 /* Dispatch the exception */
375 DispatchException eax, 3, 0, 0, 0
376
377 KiNpxNotAvailableFaultExit:
378 /* Return */
379 ExitTrap TF_SAVE_ALL
380 ENDFUNC
381
382
383 PUBLIC KiDoubleFaultAbort
384 FUNC KiDoubleFaultAbort
385 /* A zero error code is pushed */
386 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
387
388 lea rcx, MsgDoubleFault[rip]
389 mov rdx, [rbp + KTRAP_FRAME_FaultAddress]
390 mov r8, rbp
391 call qword ptr FrLdrDbgPrint[rip]
392
393 /* Bugcheck */
394 Fatal 8 // EXCEPTION_DOUBLE_FAULT
395 jmp $
396 ENDFUNC
397
398
399 PUBLIC KiNpxSegmentOverrunAbort
400 FUNC KiNpxSegmentOverrunAbort
401 /* Push pseudo error code */
402 EnterTrap TF_SAVE_ALL
403
404 /* Bugcheck */
405 Fatal EXCEPTION_NPX_OVERRUN
406
407 jmp $
408 ENDFUNC
409
410
411 PUBLIC KiInvalidTssFault
412 FUNC KiInvalidTssFault
413 /* We have an error code */
414 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416 /* Bugcheck */
417 Fatal EXCEPTION_INVALID_TSS
418 jmp $
419 ENDFUNC
420
421
422 PUBLIC KiSegmentNotPresentFault
423 FUNC KiSegmentNotPresentFault
424 /* We have an error code */
425 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
426
427 /* Bugcheck */
428 Fatal EXCEPTION_SEGMENT_NOT_PRESENT
429 jmp $
430 ENDFUNC
431
432
433 PUBLIC KiStackFault
434 FUNC KiStackFault
435 /* We have an error code */
436 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
437
438 /* Bugcheck */
439 Fatal EXCEPTION_STACK_FAULT
440 jmp $
441 ENDFUNC
442
443
444 PUBLIC KiGeneralProtectionFault
445 FUNC KiGeneralProtectionFault
446 /* We have an error code */
447 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
448
449 //TRAPINFO KiGeneralProtectionFault
450 //mov rdx, [rbp + KTRAP_FRAME_Rip]
451 //lea rcx, MsgGeneralProtFault[rip]
452 //call qword ptr FrLdrDbgPrint[rip]
453
454 /* Call the C handler */
455 mov rcx, rbp
456 call KiGeneralProtectionFaultHandler
457
458 /* Check for success */
459 test eax, eax
460 jge KiGpfExit
461
462 /* Dispatch the exception */
463 DispatchException eax, 3, 0, 0, 0
464
465 KiGpfFatal:
466
467 /* Bugcheck */
468 mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
469 mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
470 xor r8, r8
471 mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
472 sub rsp, 8
473 mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
474 call KeBugCheckWithTf
475
476 KiGpfExit:
477 /* Return */
478 /* Return */
479 ExitTrap TF_SAVE_ALL
480 ENDFUNC
481
482
483 PUBLIC KiPageFault
484 FUNC KiPageFault
485 /* We have an error code */
486 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
487
488 TRAPINFO KiPageFault
489
490 #if 0
491 lea rcx, MsgPageFault[rip]
492 mov rdx, [rbp + KTRAP_FRAME_ErrorCode]
493 mov r8, [rbp + KTRAP_FRAME_Rip]
494 mov r9, [rbp + KTRAP_FRAME_FaultAddress]
495 call qword ptr FrLdrDbgPrint[rip]
496 #endif
497
498 /* Save page fault address */
499 mov rdx, cr2
500 mov [rbp + KTRAP_FRAME_FaultAddress], rdx
501
502 /* Enable interrupts for the page fault handler */
503 sti
504
505 /* Call page fault handler */
506 mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // StoreInstruction
507 and ecx, 1
508 // rdx == Address
509 mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
510 and r8b, 1
511 mov r9, rbp // TrapInformation
512 call MmAccessFault
513
514 /* Check for success */
515 test eax, eax
516 #ifndef _WINKD_
517 jge PageFaultReturn
518 #else
519 jl PageFaultError
520
521 /* Check whether the kernel debugger has owed breakpoints to be inserted */
522 call KdSetOwedBreakpoints
523 /* We succeeded, return */
524 jmp PageFaultReturn
525
526 PageFaultError:
527 #endif
528
529 /* Disable interrupts again for the debugger */
530 cli
531
532 /* Set parameter 1 to error code */
533 mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
534
535 /* Set parameter2 to faulting address */
536 mov r10, cr2 // Param2 = faulting address
537
538 cmp eax, STATUS_ACCESS_VIOLATION
539 je AccessViolation
540 cmp eax, STATUS_GUARD_PAGE_VIOLATION
541 je SpecialCode
542 cmp eax, STATUS_STACK_OVERFLOW
543 je SpecialCode
544
545 InPageException:
546 /* Dispatch in-page exception */
547 mov r11d, eax // Param3 = Status
548 mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
549 mov edx, 3 // ParamCount
550 call InternalDispatchException
551 jmp PageFaultReturn
552
553 AccessViolation:
554 /* Use more proper status code */
555 mov eax, KI_EXCEPTION_ACCESS_VIOLATION
556
557 SpecialCode:
558 /* Setup a normal page fault exception */
559 mov edx, 2 // ParamCount
560 call InternalDispatchException
561
562 PageFaultReturn:
563 /* Return */
564 ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
565 ENDFUNC
566
567
568 PUBLIC KiFloatingErrorFault
569 FUNC KiFloatingErrorFault
570 /* Push pseudo error code */
571 EnterTrap TF_SAVE_ALL
572
573 UNIMPLEMENTED KiFloatingErrorFault
574 int 3
575
576 /* Return */
577 ExitTrap TF_SAVE_ALL
578 ENDFUNC
579
580
581 PUBLIC KiAlignmentFault
582 FUNC KiAlignmentFault
583 /* We have an error code */
584 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
585
586 /* Bugcheck */
587 Fatal EXCEPTION_ALIGNMENT_CHECK
588 jmp $
589 ENDFUNC
590
591
592 PUBLIC KiMcheckAbort
593 FUNC KiMcheckAbort
594 /* Push pseudo error code */
595 EnterTrap TF_SAVE_ALL
596
597 /* Bugcheck */
598 Fatal HEX(12)
599 jmp $
600 ENDFUNC
601
602
603 PUBLIC KiXmmException
604 FUNC KiXmmException
605 /* Push pseudo error code */
606 EnterTrap TF_SAVE_ALL
607
608 /* Call the C handler */
609 mov rcx, rbp
610 call KiXmmExceptionHandler
611
612 /* Check for success */
613 test eax, eax
614 jge KiXmmExit
615
616 /* Dispatch the exception */
617 DispatchException eax, 3, 0, 0, 0
618
619 KiXmmExit:
620 /* Return */
621 ExitTrap TF_SAVE_ALL
622 ENDFUNC
623
624
625 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
626
627 PUBLIC KiRaiseAssertion
628 FUNC KiRaiseAssertion
629 /* We have an error code */
630 EnterTrap (TF_SAVE_ALL)
631
632 /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
633 sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
634
635 /* Dispatch the exception */
636 DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
637
638 /* Return */
639 ExitTrap TF_SAVE_ALL
640 ENDFUNC
641
642
643 PUBLIC KiDebugServiceTrap
644 FUNC KiDebugServiceTrap
645 /* No error code */
646 EnterTrap TF_SAVE_ALL
647
648 TRAPINFO KiDebugServiceTrap
649
650 /* Increase Rip to skip the int3 */
651 inc qword ptr [rbp + KTRAP_FRAME_Rip]
652
653 /* Dispatch the exception (Params = service, buffer, legth) */
654 DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
655
656 /* Return */
657 ExitTrap TF_SAVE_ALL
658 ENDFUNC
659
660
661 PUBLIC KiApcInterrupt
662 .PROC KiApcInterrupt
663 /* No error code */
664 EnterTrap (TF_VOLATILES or TF_IRQL)
665
666 /* Raise to APC_LEVEL */
667 mov rax, APC_LEVEL
668 mov cr8, rax
669
670 /* End the interrupt */
671 mov dword ptr [APIC_EOI], 0
672
673 /* Enable interrupts */
674 sti
675
676 /* Call the worker routine */
677 mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
678 and cl, 1
679 mov rdx, 0 // ExceptionFrame
680 mov r8, rdx // TrapFrame
681 call KiDeliverApc
682
683 /* Disable interrupts */
684 cli
685
686 /* Lower IRQL back to PASSIVE */
687 mov rax, PASSIVE_LEVEL
688 mov cr8, rax
689
690 /* Return */
691 ExitTrap (TF_VOLATILES or TF_IRQL)
692 .ENDP
693
694 EXTERN KiRetireDpcList:PROC
695 PUBLIC KiRetireDpcListInDpcStack
696 .PROC KiRetireDpcListInDpcStack
697 push rbp
698 .pushreg rbp
699 mov rbp, rsp
700 .setframe rbp, 0
701 .endprolog
702
703 /* Switch stack and call the function */
704 mov rsp, rdx
705 sub rsp, 40
706 call KiRetireDpcList
707
708 /* Restore stack, cleanup and return */
709 mov rsp, rbp
710 pop rbp
711 ret
712 .ENDP
713
714 PUBLIC KiDpcInterrupt
715 .PROC KiDpcInterrupt
716 /* No error code */
717 EnterTrap (TF_VOLATILES or TF_IRQL)
718
719 /* Call the worker routine */
720 call KiDpcInterruptHandler
721
722 /* Return, but don't send an EOI! */
723 ExitTrap (TF_VOLATILES or TF_IRQL)
724 .ENDP
725
726
727 PUBLIC KiIpiInterrupt
728 .PROC KiIpiInterrupt
729 /* No error code */
730 EnterTrap (TF_VOLATILES or TF_IRQL)
731
732 /* Raise to IPI_LEVEL */
733 mov rax, IPI_LEVEL
734 mov cr8, rax
735
736 /* End the interrupt */
737 mov dword ptr [APIC_EOI], 0
738
739 int 3
740
741 /* Return */
742 ExitTrap (TF_VOLATILES or TF_IRQL)
743 .ENDP
744
745
746 PUBLIC KiUnexpectedInterrupt
747 FUNC KiUnexpectedInterrupt
748 /* The error code is the vector */
749 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
750
751 #if 0
752 /* Set bugcheck parameters */
753 mov ecx, TRAP_CAUSE_UNKNOWN
754 mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
755 mov r8, 0 // The unknown floating-point exception
756 mov r9, 0 // The enabled and asserted status bits
757 sub rsp, 8
758 mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
759 call KeBugCheckWithTf
760 jmp $
761 #endif
762 /* Return */
763 ExitTrap TF_SAVE_ALL
764 ENDFUNC
765
766 PUBLIC KiInterruptDispatch
767 FUNC KiInterruptDispatch
768 /* The error code is a pointer to the interrupt object's code */
769 EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
770
771 /* Increase interrupt count */
772 inc dword ptr gs:[PcInterruptCount];
773
774 /* Load the address of the interrupt object into rcx */
775 mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
776
777 /* Substract offset of the DispatchCode member plus 6 for the call instruction */
778 sub rcx, KINTERRUPT_DispatchCode + 6
779
780 /* Raise IRQL to SynchronizeIrql */
781 movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
782 mov cr8, rax
783
784 #ifdef CONFIG_SMP
785 /* Acquire interrupt lock */
786 mov r8, [rcx + KINTERRUPT_ActualLock]
787
788 //KxAcquireSpinLock(Interrupt->ActualLock);
789 #endif
790
791 /* Call the ISR */
792 mov rdx, [rcx + KINTERRUPT_ServiceContext]
793 call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
794
795 #ifdef CONFIG_SMP
796 /* Release interrupt lock */
797 //KxReleaseSpinLock(Interrupt->ActualLock);
798 #endif
799
800 /* Go back to old irql */
801 movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
802 mov cr8, rax
803
804 /* Return */
805 ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
806 ENDFUNC
807
808
809 #define MAX_SYSCALL_PARAM_SIZE (16 * 8)
810 #define HOME_SIZE 6*8
811 #define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
812
813 EXTERN KiSystemCallHandler:PROC
814
815 /*! \name KiSystemCallEntry64
816 *
817 * \brief This is the entrypoint for syscalls from 64bit user mode
818 *
819 * \param rax - The system call number
820 * \param rcx - User mode return address, set by the syscall instruction
821 * \param rdx,r8,r9 - Parameters 2-4 to the service function
822 * \param r10 - Parameter 1 to the service function
823 * \param r11 - RFLAGS saved by the syscall instruction
824 *--*/
825 PUBLIC KiSystemCallEntry64
826 .PROC KiSystemCallEntry64
827
828 /* Old stack pointer is in rcx, lie and say we saved it in rbp */
829 .setframe rbp, 0
830 .endprolog
831
832 /* Swap gs to kernel, so we can access the PCR */
833 swapgs
834
835 /* Save the user mode rsp in the PCR */
836 mov gs:[PcUserRsp], rsp
837
838 /* Get the kernel stack from the PCR */
839 mov rsp, gs:[PcRspBase]
840
841 /* Allocate a TRAP_FRAME and space for parameters */
842 sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
843 #if DBG
844 /* Save rbp and load it with the old stack pointer */
845 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
846 mov rbp, gs:[PcUserRsp]
847 #endif
848
849 /* Save important volatiles in the trap frame */
850 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
851 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
852 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
853 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
854
855 /* Set sane segments */
856 mov ax, (KGDT64_R3_DATA or RPL_MASK)
857 mov ds, ax
858 mov es, ax
859
860 /* Call the C-handler (will enable interrupts) */
861 lea rcx, [rsp + SYSCALL_ALLOCATION]
862 call KiSystemCallHandler
863
864 /* Deallocate the handlers home stack frame */
865 add rsp, HOME_SIZE
866
867 /* The return value is the address of the Nt-function */
868 mov rcx, [rsp + 0]
869 mov rdx, [rsp + 8]
870 mov r8, [rsp + 16]
871 mov r9, [rsp + 24]
872 call rax
873
874 #if DBG
875 /* Restore rbp */
876 mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
877 #endif
878
879 /* Disable interrupts for return */
880 cli
881
882 /* Restore old trap frame */
883 mov rcx, gs:[PcCurrentThread]
884 mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
885 mov [rcx + KTHREAD_TrapFrame], rdx
886
887 /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
888 mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
889 mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
890
891 /* Load user mode stack (It was copied to the trap frame) */
892 mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
893
894 /* Swap gs back to user */
895 swapgs
896
897 /* return to user mode */
898 .byte HEX(48) // REX prefix to return to long mode
899 sysret
900 .ENDP
901
902
903 PUBLIC KiSystemCallEntry32
904 KiSystemCallEntry32:
905 swapgs
906 int 3
907
908
909 PUBLIC KiZwSystemService
910 FUNC KiZwSystemService
911 push rbp
912 .pushreg rbp
913 sub rsp, KTRAP_FRAME_LENGTH
914 .allocstack KTRAP_FRAME_LENGTH
915 mov [rsp + KTRAP_FRAME_Rsi], rsi
916 .savereg rsi, KTRAP_FRAME_Rsi
917 mov [rsp + KTRAP_FRAME_Rdi], rdi
918 .savereg rdi, KTRAP_FRAME_Rdi
919 mov rbp, rsp
920 .setframe rbp, 0
921 .endprolog
922
923 /* Get current thread */
924 mov r11, gs:[PcCurrentThread]
925
926 /* Save the old trap frame in TrapFrame.Rdx */
927 mov rdi, [r11 + KTHREAD_TrapFrame]
928 mov [rbp + KTRAP_FRAME_Rdx], rdi
929
930 /* Set the new trap frame and previous mode */
931 mov [r11 + ThTrapFrame], rbp
932 mov byte ptr [r11 + KTHREAD_PreviousMode], 0
933
934 /* allocate space for parameters */
935 sub rsp, r10
936 and rsp, HEX(0fffffffffffffff0)
937
938 /* Save rcx */
939 mov [rbp + KTRAP_FRAME_Rcx], rcx
940
941 /* copy parameters to the new location */
942 lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
943 lea rdi, [rsp]
944 mov rcx, r10
945 shr rcx, 3
946 rep movsq
947
948 /* Restore rcx */
949 mov rcx, [rbp + KTRAP_FRAME_Rcx]
950
951 /* Call the service function */
952 call rax
953
954 /* Restore the old trap frame */
955 mov r11, gs:[PcCurrentThread]
956 mov rsi, [rsp + KTRAP_FRAME_Rdx]
957 mov [r11 + KTHREAD_TrapFrame], rsi
958
959 /* Restore rdi and rsi */
960 mov rsi, [rbp + KTRAP_FRAME_Rsi]
961 mov rdi, [rbp + KTRAP_FRAME_Rdi]
962
963 /* Cleanup the stack and return */
964 lea rsp, [rbp + KTRAP_FRAME_LENGTH]
965 pop rbp
966 ret
967
968 ENDFUNC
969
970
971 KiExitToUserApc:
972 int 3
973
974 /*!
975 * VOID
976 * DECLSPEC_NORETURN
977 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
978 */
979 PUBLIC KiServiceExit
980 KiServiceExit:
981 mov [rcx + KTRAP_FRAME_Rax], rdx
982 mov rbp, rcx
983 mov rsp, rcx
984
985 /* Return */
986 //ExitTrap TF_SAVE_ALL
987
988 /*!
989 * VOID
990 * DECLSPEC_NORETURN
991 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
992 */
993 PUBLIC KiServiceExit2
994 .PROC KiServiceExit2
995 .ENDPROLOG
996
997 mov rbp, rcx
998 mov rsp, rcx
999
1000 /* Return */
1001 ExitTrap TF_SAVE_ALL
1002 .ENDP
1003
1004 PUBLIC KiInitializeSegments
1005 KiInitializeSegments:
1006 mov ax, KGDT64_R3_DATA or RPL_MASK
1007 mov gs, ax
1008 swapgs
1009 mov gs, ax
1010 ret
1011
1012
1013 #ifdef _MSC_VER
1014 #undef lgdt
1015 #undef lidt
1016
1017 //void __lgdt(void *Source);
1018 PUBLIC __lgdt
1019 __lgdt:
1020 lgdt fword ptr [rcx]
1021 ret
1022
1023 //void __sgdt(void *Destination);
1024 PUBLIC __sgdt
1025 __sgdt:
1026 sgdt fword ptr [rcx]
1027 ret
1028
1029 // void __lldt(unsigned short Value)
1030 PUBLIC __lldt
1031 __lldt:
1032 lldt cx
1033 ret
1034
1035 //void __sldt(void *Destination);
1036 PUBLIC __sldt
1037 __sldt:
1038 sldt word ptr [rcx]
1039 ret
1040
1041 //void __ltr(unsigned short Source);
1042 PUBLIC __ltr
1043 __ltr:
1044 ltr cx
1045 ret
1046
1047 //void __str(unsigned short *Destination);
1048 PUBLIC __str
1049 __str:
1050 str word ptr [rcx]
1051 ret
1052
1053 PUBLIC __swapgs
1054 __swapgs:
1055 swapgs
1056 ret
1057
1058 #endif
1059
1060 END