[NTOS:MM] Pass page fault code to MmAccessFault
[reactos.git] / ntoskrnl / ke / amd64 / trap.S
1 /*
2 * FILE: ntoskrnl/ke/amd64/trap.S
3 * COPYRIGHT: See COPYING in the top level directory
4 * PURPOSE: System Traps, Entrypoints and Exitpoints
5 * PROGRAMMER: Timo Kreuzer (timo.kreuzer@reactos.org)
6 */
7
8 /* INCLUDES ******************************************************************/
9
10 #include <asm.inc>
11 #include <ksamd64.inc>
12 #include <trapamd64.inc>
13
14 EXTERN KiDispatchException:PROC
15 EXTERN FrLdrDbgPrint:DWORD
16 EXTERN KeBugCheckWithTf:PROC
17 EXTERN MmAccessFault:PROC
18 EXTERN KiSystemFatalException:PROC
19 EXTERN KiNpxNotAvailableFaultHandler:PROC
20 EXTERN KiGeneralProtectionFaultHandler:PROC
21 EXTERN KiXmmExceptionHandler:PROC
22 EXTERN KiDeliverApc:PROC
23 EXTERN KiDpcInterruptHandler:PROC
24
25 #ifdef _WINKD_
26 EXTERN KdSetOwedBreakpoints:PROC
27 #endif
28
29
30 /* GLOBALS *******************************************************************/
31
32 .data
33
34 PUBLIC MsgUnimplemented
35 MsgUnimplemented:
36 .asciz "WARNING: %s at %s:%d is UNIMPLEMENTED!\n"
37
38 MsgPageFault:
39 .asciz "Page fault! Code = 0x%x, RIP = %p, FaultingAddress = %p\n"
40
41 MsgGeneralProtFault:
42 .asciz "General protection fault at %p!\n"
43
44 MsgBreakpointTrap:
45 .asciz "BreakpointTrap at %p\n"
46
47 MsgUnexpectedInterrupt:
48 .asciz "UnexpectedInterrupt Vector=0x%02lx\n"
49
50 MsgInvalidOpcodeFault:
51 .asciz "Invalid opcode fault at %p!\n"
52
53 MsgDoubleFault:
54 .asciz "Double fault at %p, rbp=%p!\n"
55
56 MsgTrapInfo:
57 .asciz "Trap: %s at %p\n"
58
59 MACRO(TRAPINFO, func)
60 LOCAL label1, label2
61 #if 0
62 jmp label2
63 label1: .asciz "\func"
64 label2:
65 push rax
66 push rcx
67 push rdx
68 push r8
69 push r9
70 push r10
71 push r11
72 sub rsp, 32
73 lea rcx, MsgTrapInfo[rip]
74 lea rdx, 1b[rip]
75 mov r8, [rbp + KTRAP_FRAME_Rip]
76 call qword ptr FrLdrDbgPrint[rip]
77 pop r11
78 pop r10
79 pop r9
80 pop r8
81 pop rdx
82 pop rcx
83 pop rax
84 add rsp, 32
85 #endif
86 ENDM
87
88 /* Helper Macros *************************************************************/
89
90 MACRO(DispatchException, Status, Number, P1, P2, P3)
91 mov eax, Status
92 mov edx, Number
93 mov r9, P1
94 mov r10, P2
95 mov r11, P3
96 call InternalDispatchException
97 ENDM
98
99 MACRO(Fatal, BugcheckCode)
100 /* Bugcheck */
101 mov ecx, BugcheckCode
102 mov rdx, rbp
103 call KiSystemFatalException
104 ENDM
105
106
107 /* FUNCTIONS *****************************************************************/
108
109 .code64
110
111 ALIGN 8
112
113 MACRO(UnexpectedVectorStub, Vector)
114 /* This nop is to make the relative jmp address 4 bytes aligned and to
115 make the whole code 8 bytes long */
116 nop
117 /* This is a push instruction with 8bit operand. Since the instruction
118 sign extends the value to 32 bits, we need to offset it */
119 PUBLIC KxUnexpectedInterrupt&Vector
120 KxUnexpectedInterrupt&Vector:
121 push (Vector - 128)
122 jmp KiUnexpectedInterrupt
123 ENDM
124
125 PUBLIC KiUnexpectedRange
126 KiUnexpectedRange:
127 Vector = 0
128 REPEAT 256
129 UnexpectedVectorStub %Vector
130 Vector = Vector+1
131 ENDR
132 PUBLIC KiUnexpectedRangeEnd
133 KiUnexpectedRangeEnd:
134
135 PUBLIC KiInterruptDispatchTemplate
136 KiInterruptDispatchTemplate:
137 /* This instruction pushes the return address on the stack, which is the
138 address of the interrupt object's DispatchCode member, then jumps
139 to the address stored in the interrupt object's DispatchAddress member */
140 call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
141
142
143 // rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
144 FUNC InternalDispatchException
145
146 /* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
147 sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
148 .allocstack (EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH)
149 .endprolog
150
151 /* Set up EXCEPTION_RECORD */
152 lea rcx, [rsp + KEXCEPTION_FRAME_LENGTH]
153 mov [rcx + EXCEPTION_RECORD_ExceptionCode], eax
154 xor rax, rax
155 mov [rcx + EXCEPTION_RECORD_ExceptionFlags], eax
156 mov [rcx + EXCEPTION_RECORD_ExceptionRecord], rax
157 mov rax, [rbp + KTRAP_FRAME_Rip]
158 mov [rcx + EXCEPTION_RECORD_ExceptionAddress], rax
159 mov [rcx + EXCEPTION_RECORD_NumberParameters], edx
160 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(00)], r9
161 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(08)], r10
162 mov [rcx + EXCEPTION_RECORD_ExceptionInformation + HEX(10)], r11
163
164 /* Set up KEXCEPTION_FRAME */
165 mov rax, [rbp + KTRAP_FRAME_Rbp]
166 mov [rsp + KEXCEPTION_FRAME_Rbp], rax
167 mov [rsp + KEXCEPTION_FRAME_Rbx], rbx
168 mov [rsp + KEXCEPTION_FRAME_Rdi], rdi
169 mov [rsp + KEXCEPTION_FRAME_Rsi], rsi
170 mov [rsp + KEXCEPTION_FRAME_R12], r12
171 mov [rsp + KEXCEPTION_FRAME_R13], r13
172 mov [rsp + KEXCEPTION_FRAME_R14], r14
173 mov [rsp + KEXCEPTION_FRAME_R15], r15
174 movdqa [rsp + KEXCEPTION_FRAME_Xmm6], xmm6
175 movdqa [rsp + KEXCEPTION_FRAME_Xmm7], xmm7
176 movdqa [rsp + KEXCEPTION_FRAME_Xmm8], xmm8
177 movdqa [rsp + KEXCEPTION_FRAME_Xmm9], xmm9
178 movdqa [rsp + KEXCEPTION_FRAME_Xmm10], xmm10
179 movdqa [rsp + KEXCEPTION_FRAME_Xmm11], xmm11
180 movdqa [rsp + KEXCEPTION_FRAME_Xmm12], xmm12
181 movdqa [rsp + KEXCEPTION_FRAME_Xmm13], xmm13
182 movdqa [rsp + KEXCEPTION_FRAME_Xmm14], xmm14
183 movdqa [rsp + KEXCEPTION_FRAME_Xmm15], xmm15
184 mov qword ptr [rsp + KEXCEPTION_FRAME_Return], 0
185
186 /* Call KiDispatchException */
187 // rcx already points to ExceptionRecord
188 mov rdx, rsp // ExceptionFrame
189 mov r8, rbp // TrapFrame
190 mov r9b, [r8 + KTRAP_FRAME_PreviousMode] // PreviousMode
191 mov byte ptr [rsp + KEXCEPTION_FRAME_P5], 1 // FirstChance
192 call KiDispatchException
193
194 /* Restore registers */
195 mov r12, [rsp + KEXCEPTION_FRAME_R12]
196 mov r13, [rsp + KEXCEPTION_FRAME_R13]
197 mov r14, [rsp + KEXCEPTION_FRAME_R14]
198 mov r15, [rsp + KEXCEPTION_FRAME_R15]
199 movdqa xmm6, [rsp + KEXCEPTION_FRAME_Xmm6]
200 movdqa xmm7, [rsp + KEXCEPTION_FRAME_Xmm7]
201 movdqa xmm8, [rsp + KEXCEPTION_FRAME_Xmm8]
202 movdqa xmm9, [rsp + KEXCEPTION_FRAME_Xmm9]
203 movdqa xmm10, [rsp + KEXCEPTION_FRAME_Xmm10]
204 movdqa xmm11, [rsp + KEXCEPTION_FRAME_Xmm11]
205 movdqa xmm12, [rsp + KEXCEPTION_FRAME_Xmm12]
206 movdqa xmm13, [rsp + KEXCEPTION_FRAME_Xmm13]
207 movdqa xmm14, [rsp + KEXCEPTION_FRAME_Xmm14]
208 movdqa xmm15, [rsp + KEXCEPTION_FRAME_Xmm15]
209
210 add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
211 ret
212 ENDFUNC
213
214
215 /* CPU EXCEPTION HANDLERS ****************************************************/
216
217 PUBLIC KiDivideErrorFault
218 FUNC KiDivideErrorFault
219 /* Push pseudo error code */
220 EnterTrap TF_SAVE_ALL
221
222 /* Enable interrupts */
223 sti
224
225 /* Dispatch the exception */
226 DispatchException STATUS_INTEGER_DIVIDE_BY_ZERO, 0, 0, 0, 0
227
228 /* Return */
229 ExitTrap TF_SAVE_ALL
230 ENDFUNC
231
232
233 PUBLIC KiDebugTrapOrFault
234 FUNC KiDebugTrapOrFault
235 /* Push pseudo error code */
236 EnterTrap TF_SAVE_ALL
237
238 TRAPINFO KiDebugTrapOrFault
239
240 /* Check if the frame was from kernelmode */
241 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
242 jz KiDebugTrapOrFaultKMode
243
244 /* Enable interrupts for user-mode */
245 sti
246
247 KiDebugTrapOrFaultKMode:
248 /* Dispatch the exception */
249 DispatchException STATUS_SINGLE_STEP, 0, 0, 0, 0
250
251 /* Return */
252 ExitTrap TF_SAVE_ALL
253 ENDFUNC
254
255
256 PUBLIC KiNmiInterrupt
257 FUNC KiNmiInterrupt
258 /* Push pseudo error code */
259 EnterTrap TF_SAVE_ALL
260
261 UNIMPLEMENTED KiNmiInterrupt
262 int 3
263
264 /* Return */
265 ExitTrap TF_SAVE_ALL
266 ENDFUNC
267
268
269 PUBLIC KiBreakpointTrap
270 FUNC KiBreakpointTrap
271 /* Push pseudo error code */
272 EnterTrap TF_SAVE_ALL
273
274 /* Check if the frame was from kernelmode */
275 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
276 jz KiBreakpointTrapKMode
277
278 /* Enable interrupts for user-mode */
279 sti
280
281 KiBreakpointTrapKMode:
282 /* Dispatch the exception */
283 DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
284
285 /* Return */
286 ExitTrap TF_SAVE_ALL
287 ENDFUNC
288
289
290 PUBLIC KiOverflowTrap
291 FUNC KiOverflowTrap
292 /* Push pseudo error code */
293 EnterTrap TF_SAVE_ALL
294
295 /* Enable interrupts */
296 sti
297
298 /* Dispatch the exception */
299 DispatchException STATUS_INTEGER_OVERFLOW, 3, 0, 0, 0
300
301 /* Return */
302 ExitTrap TF_SAVE_ALL
303 ENDFUNC
304
305
306 PUBLIC KiBoundFault
307 FUNC KiBoundFault
308 /* Push pseudo error code */
309 EnterTrap TF_SAVE_ALL
310
311 /* Check if the frame was from kernelmode */
312 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
313 jnz KiBoundFaultUserMode
314
315 /* Bugcheck */
316 Fatal EXCEPTION_BOUND_CHECK
317
318 KiBoundFaultUserMode:
319 /* Enable interrupts for user-mode */
320 sti
321
322 /* Dispatch the exception */
323 DispatchException STATUS_ARRAY_BOUNDS_EXCEEDED, 0, 0, 0, 0
324
325 /* Return */
326 ExitTrap TF_SAVE_ALL
327 ENDFUNC
328
329
330 PUBLIC KiInvalidOpcodeFault
331 FUNC KiInvalidOpcodeFault
332 /* Push pseudo error code */
333 EnterTrap TF_SAVE_ALL
334
335 TRAPINFO KiInvalidOpcodeFault
336
337 mov rdx, [rbp + KTRAP_FRAME_Rip]
338 lea rcx, MsgInvalidOpcodeFault[rip]
339 call qword ptr FrLdrDbgPrint[rip]
340
341 /* Enable interrupts */
342 sti
343
344 /* Check if the frame was from kernelmode */
345 test word ptr [rbp + KTRAP_FRAME_SegCs], 3
346 jz KiInvalidOpcodeKernel
347
348 // FIXME: handle STATUS_INVALID_LOCK_SEQUENCE
349
350 KiInvalidOpcodeKernel:
351 /* Kernel mode fault */
352
353 /* Dispatch the exception */
354 DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
355
356 /* Return */
357 ExitTrap TF_SAVE_ALL
358 ENDFUNC
359
360
361 PUBLIC KiNpxNotAvailableFault
362 FUNC KiNpxNotAvailableFault
363 /* Push pseudo error code */
364 EnterTrap TF_SAVE_ALL
365
366 /* Call the C handler */
367 mov rcx, rbp
368 call KiNpxNotAvailableFaultHandler
369
370 /* Check the return status code */
371 test eax, eax
372 jz KiNpxNotAvailableFaultExit
373
374 /* Dispatch the exception */
375 DispatchException eax, 3, 0, 0, 0
376
377 KiNpxNotAvailableFaultExit:
378 /* Return */
379 ExitTrap TF_SAVE_ALL
380 ENDFUNC
381
382
383 PUBLIC KiDoubleFaultAbort
384 FUNC KiDoubleFaultAbort
385 /* A zero error code is pushed */
386 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
387
388 lea rcx, MsgDoubleFault[rip]
389 mov rdx, [rbp + KTRAP_FRAME_FaultAddress]
390 mov r8, rbp
391 call qword ptr FrLdrDbgPrint[rip]
392
393 /* Bugcheck */
394 Fatal 8 // EXCEPTION_DOUBLE_FAULT
395 jmp $
396 ENDFUNC
397
398
399 PUBLIC KiNpxSegmentOverrunAbort
400 FUNC KiNpxSegmentOverrunAbort
401 /* Push pseudo error code */
402 EnterTrap TF_SAVE_ALL
403
404 /* Bugcheck */
405 Fatal EXCEPTION_NPX_OVERRUN
406
407 jmp $
408 ENDFUNC
409
410
411 PUBLIC KiInvalidTssFault
412 FUNC KiInvalidTssFault
413 /* We have an error code */
414 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
415
416 /* Bugcheck */
417 Fatal EXCEPTION_INVALID_TSS
418 jmp $
419 ENDFUNC
420
421
422 PUBLIC KiSegmentNotPresentFault
423 FUNC KiSegmentNotPresentFault
424 /* We have an error code */
425 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
426
427 /* Bugcheck */
428 Fatal EXCEPTION_SEGMENT_NOT_PRESENT
429 jmp $
430 ENDFUNC
431
432
433 PUBLIC KiStackFault
434 FUNC KiStackFault
435 /* We have an error code */
436 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
437
438 /* Bugcheck */
439 Fatal EXCEPTION_STACK_FAULT
440 jmp $
441 ENDFUNC
442
443
444 PUBLIC KiGeneralProtectionFault
445 FUNC KiGeneralProtectionFault
446 /* We have an error code */
447 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
448
449 //TRAPINFO KiGeneralProtectionFault
450 //mov rdx, [rbp + KTRAP_FRAME_Rip]
451 //lea rcx, MsgGeneralProtFault[rip]
452 //call qword ptr FrLdrDbgPrint[rip]
453
454 /* Call the C handler */
455 mov rcx, rbp
456 call KiGeneralProtectionFaultHandler
457
458 /* Check for success */
459 test eax, eax
460 jge KiGpfExit
461
462 /* Dispatch the exception */
463 DispatchException eax, 3, 0, 0, 0
464
465 KiGpfFatal:
466
467 /* Bugcheck */
468 mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
469 mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
470 xor r8, r8
471 mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
472 sub rsp, 8
473 mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
474 call KeBugCheckWithTf
475
476 KiGpfExit:
477 /* Return */
478 /* Return */
479 ExitTrap TF_SAVE_ALL
480 ENDFUNC
481
482
483 PUBLIC KiPageFault
484 FUNC KiPageFault
485 /* We have an error code */
486 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
487
488 TRAPINFO KiPageFault
489
490 #if 0
491 lea rcx, MsgPageFault[rip]
492 mov rdx, [rbp + KTRAP_FRAME_ErrorCode]
493 mov r8, [rbp + KTRAP_FRAME_Rip]
494 mov r9, [rbp + KTRAP_FRAME_FaultAddress]
495 call qword ptr FrLdrDbgPrint[rip]
496 #endif
497
498 /* Save page fault address */
499 mov rdx, cr2
500 mov [rbp + KTRAP_FRAME_FaultAddress], rdx
501
502 /* Enable interrupts for the page fault handler */
503 sti
504
505 /* Call page fault handler */
506 mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
507 // rdx == Address
508 mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
509 and r8b, 1
510 mov r9, rbp // TrapInformation
511 call MmAccessFault
512
513 /* Check for success */
514 test eax, eax
515 #ifndef _WINKD_
516 jge PageFaultReturn
517 #else
518 jl PageFaultError
519
520 /* Check whether the kernel debugger has owed breakpoints to be inserted */
521 call KdSetOwedBreakpoints
522 /* We succeeded, return */
523 jmp PageFaultReturn
524
525 PageFaultError:
526 #endif
527
528 /* Disable interrupts again for the debugger */
529 cli
530
531 /* Set parameter 1 to error code */
532 mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
533
534 /* Set parameter2 to faulting address */
535 mov r10, cr2 // Param2 = faulting address
536
537 cmp eax, STATUS_ACCESS_VIOLATION
538 je AccessViolation
539 cmp eax, STATUS_GUARD_PAGE_VIOLATION
540 je SpecialCode
541 cmp eax, STATUS_STACK_OVERFLOW
542 je SpecialCode
543
544 InPageException:
545 /* Dispatch in-page exception */
546 mov r11d, eax // Param3 = Status
547 mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
548 mov edx, 3 // ParamCount
549 call InternalDispatchException
550 jmp PageFaultReturn
551
552 AccessViolation:
553 /* Use more proper status code */
554 mov eax, KI_EXCEPTION_ACCESS_VIOLATION
555
556 SpecialCode:
557 /* Setup a normal page fault exception */
558 mov edx, 2 // ParamCount
559 call InternalDispatchException
560
561 PageFaultReturn:
562 /* Return */
563 ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
564 ENDFUNC
565
566
567 PUBLIC KiFloatingErrorFault
568 FUNC KiFloatingErrorFault
569 /* Push pseudo error code */
570 EnterTrap TF_SAVE_ALL
571
572 UNIMPLEMENTED KiFloatingErrorFault
573 int 3
574
575 /* Return */
576 ExitTrap TF_SAVE_ALL
577 ENDFUNC
578
579
580 PUBLIC KiAlignmentFault
581 FUNC KiAlignmentFault
582 /* We have an error code */
583 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
584
585 /* Bugcheck */
586 Fatal EXCEPTION_ALIGNMENT_CHECK
587 jmp $
588 ENDFUNC
589
590
591 PUBLIC KiMcheckAbort
592 FUNC KiMcheckAbort
593 /* Push pseudo error code */
594 EnterTrap TF_SAVE_ALL
595
596 /* Bugcheck */
597 Fatal HEX(12)
598 jmp $
599 ENDFUNC
600
601
602 PUBLIC KiXmmException
603 FUNC KiXmmException
604 /* Push pseudo error code */
605 EnterTrap TF_SAVE_ALL
606
607 /* Call the C handler */
608 mov rcx, rbp
609 call KiXmmExceptionHandler
610
611 /* Check for success */
612 test eax, eax
613 jge KiXmmExit
614
615 /* Dispatch the exception */
616 DispatchException eax, 3, 0, 0, 0
617
618 KiXmmExit:
619 /* Return */
620 ExitTrap TF_SAVE_ALL
621 ENDFUNC
622
623
624 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
625
626 PUBLIC KiRaiseAssertion
627 FUNC KiRaiseAssertion
628 /* We have an error code */
629 EnterTrap (TF_SAVE_ALL)
630
631 /* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
632 sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
633
634 /* Dispatch the exception */
635 DispatchException STATUS_ASSERTION_FAILURE, 0, 0, 0, 0
636
637 /* Return */
638 ExitTrap TF_SAVE_ALL
639 ENDFUNC
640
641
642 PUBLIC KiDebugServiceTrap
643 FUNC KiDebugServiceTrap
644 /* No error code */
645 EnterTrap TF_SAVE_ALL
646
647 TRAPINFO KiDebugServiceTrap
648
649 /* Increase Rip to skip the int3 */
650 inc qword ptr [rbp + KTRAP_FRAME_Rip]
651
652 /* Dispatch the exception (Params = service, buffer, legth) */
653 DispatchException STATUS_BREAKPOINT, 3, [rbp+KTRAP_FRAME_Rax], [rbp+KTRAP_FRAME_Rcx], [rbp+KTRAP_FRAME_Rdx]
654
655 /* Return */
656 ExitTrap TF_SAVE_ALL
657 ENDFUNC
658
659
660 PUBLIC KiApcInterrupt
661 .PROC KiApcInterrupt
662 /* No error code */
663 EnterTrap (TF_VOLATILES or TF_IRQL)
664
665 /* Raise to APC_LEVEL */
666 mov rax, APC_LEVEL
667 mov cr8, rax
668
669 /* End the interrupt */
670 mov dword ptr [APIC_EOI], 0
671
672 /* Enable interrupts */
673 sti
674
675 /* Call the worker routine */
676 mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
677 and cl, 1
678 mov rdx, 0 // ExceptionFrame
679 mov r8, rdx // TrapFrame
680 call KiDeliverApc
681
682 /* Disable interrupts */
683 cli
684
685 /* Lower IRQL back to PASSIVE */
686 mov rax, PASSIVE_LEVEL
687 mov cr8, rax
688
689 /* Return */
690 ExitTrap (TF_VOLATILES or TF_IRQL)
691 .ENDP
692
693 EXTERN KiRetireDpcList:PROC
694 PUBLIC KiRetireDpcListInDpcStack
695 .PROC KiRetireDpcListInDpcStack
696 push rbp
697 .pushreg rbp
698 mov rbp, rsp
699 .setframe rbp, 0
700 .endprolog
701
702 /* Switch stack and call the function */
703 mov rsp, rdx
704 sub rsp, 40
705 call KiRetireDpcList
706
707 /* Restore stack, cleanup and return */
708 mov rsp, rbp
709 pop rbp
710 ret
711 .ENDP
712
713 PUBLIC KiDpcInterrupt
714 .PROC KiDpcInterrupt
715 /* No error code */
716 EnterTrap (TF_VOLATILES or TF_IRQL)
717
718 /* Call the worker routine */
719 call KiDpcInterruptHandler
720
721 /* Return, but don't send an EOI! */
722 ExitTrap (TF_VOLATILES or TF_IRQL)
723 .ENDP
724
725
726 PUBLIC KiIpiInterrupt
727 .PROC KiIpiInterrupt
728 /* No error code */
729 EnterTrap (TF_VOLATILES or TF_IRQL)
730
731 /* Raise to IPI_LEVEL */
732 mov rax, IPI_LEVEL
733 mov cr8, rax
734
735 /* End the interrupt */
736 mov dword ptr [APIC_EOI], 0
737
738 int 3
739
740 /* Return */
741 ExitTrap (TF_VOLATILES or TF_IRQL)
742 .ENDP
743
744
745 PUBLIC KiUnexpectedInterrupt
746 FUNC KiUnexpectedInterrupt
747 /* The error code is the vector */
748 EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
749
750 #if 0
751 /* Set bugcheck parameters */
752 mov ecx, TRAP_CAUSE_UNKNOWN
753 mov rdx, [rbp + KTRAP_FRAME_ErrorCode] // the vector
754 mov r8, 0 // The unknown floating-point exception
755 mov r9, 0 // The enabled and asserted status bits
756 sub rsp, 8
757 mov [rbp + KTRAP_FRAME_P5 + 8], rbp // trap frame
758 call KeBugCheckWithTf
759 jmp $
760 #endif
761 /* Return */
762 ExitTrap TF_SAVE_ALL
763 ENDFUNC
764
765 PUBLIC KiInterruptDispatch
766 FUNC KiInterruptDispatch
767 /* The error code is a pointer to the interrupt object's code */
768 EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
769
770 /* Increase interrupt count */
771 inc dword ptr gs:[PcInterruptCount];
772
773 /* Load the address of the interrupt object into rcx */
774 mov rcx, [rbp + KTRAP_FRAME_ErrorCode]
775
776 /* Substract offset of the DispatchCode member plus 6 for the call instruction */
777 sub rcx, KINTERRUPT_DispatchCode + 6
778
779 /* Raise IRQL to SynchronizeIrql */
780 movzx rax, byte ptr [rcx + KINTERRUPT_SynchronizeIrql]
781 mov cr8, rax
782
783 #ifdef CONFIG_SMP
784 /* Acquire interrupt lock */
785 mov r8, [rcx + KINTERRUPT_ActualLock]
786
787 //KxAcquireSpinLock(Interrupt->ActualLock);
788 #endif
789
790 /* Call the ISR */
791 mov rdx, [rcx + KINTERRUPT_ServiceContext]
792 call qword ptr [rcx + KINTERRUPT_ServiceRoutine]
793
794 #ifdef CONFIG_SMP
795 /* Release interrupt lock */
796 //KxReleaseSpinLock(Interrupt->ActualLock);
797 #endif
798
799 /* Go back to old irql */
800 movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
801 mov cr8, rax
802
803 /* Return */
804 ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
805 ENDFUNC
806
807
808 #define MAX_SYSCALL_PARAM_SIZE (16 * 8)
809 #define HOME_SIZE 6*8
810 #define SYSCALL_ALLOCATION (MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
811
812 EXTERN KiSystemCallHandler:PROC
813
814 /*! \name KiSystemCallEntry64
815 *
816 * \brief This is the entrypoint for syscalls from 64bit user mode
817 *
818 * \param rax - The system call number
819 * \param rcx - User mode return address, set by the syscall instruction
820 * \param rdx,r8,r9 - Parameters 2-4 to the service function
821 * \param r10 - Parameter 1 to the service function
822 * \param r11 - RFLAGS saved by the syscall instruction
823 *--*/
824 PUBLIC KiSystemCallEntry64
825 .PROC KiSystemCallEntry64
826
827 /* Old stack pointer is in rcx, lie and say we saved it in rbp */
828 .setframe rbp, 0
829 .endprolog
830
831 /* Swap gs to kernel, so we can access the PCR */
832 swapgs
833
834 /* Save the user mode rsp in the PCR */
835 mov gs:[PcUserRsp], rsp
836
837 /* Get the kernel stack from the PCR */
838 mov rsp, gs:[PcRspBase]
839
840 /* Allocate a TRAP_FRAME and space for parameters */
841 sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE)
842 #if DBG
843 /* Save rbp and load it with the old stack pointer */
844 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + HOME_SIZE + KTRAP_FRAME_Rbp], rbp
845 mov rbp, gs:[PcUserRsp]
846 #endif
847
848 /* Save important volatiles in the trap frame */
849 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
850 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rcx
851 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], r10
852 mov [rsp + HOME_SIZE + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], r11
853
854 /* Set sane segments */
855 mov ax, (KGDT64_R3_DATA or RPL_MASK)
856 mov ds, ax
857 mov es, ax
858
859 /* Call the C-handler (will enable interrupts) */
860 lea rcx, [rsp + SYSCALL_ALLOCATION]
861 call KiSystemCallHandler
862
863 /* Deallocate the handlers home stack frame */
864 add rsp, HOME_SIZE
865
866 /* The return value is the address of the Nt-function */
867 mov rcx, [rsp + 0]
868 mov rdx, [rsp + 8]
869 mov r8, [rsp + 16]
870 mov r9, [rsp + 24]
871 call rax
872
873 #if DBG
874 /* Restore rbp */
875 mov rbp, [rsp + SYSCALL_ALLOCATION + KTRAP_FRAME_Rbp]
876 #endif
877
878 /* Disable interrupts for return */
879 cli
880
881 /* Restore old trap frame */
882 mov rcx, gs:[PcCurrentThread]
883 mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
884 mov [rcx + KTHREAD_TrapFrame], rdx
885
886 /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
887 mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
888 mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11]
889
890 /* Load user mode stack (It was copied to the trap frame) */
891 mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
892
893 /* Swap gs back to user */
894 swapgs
895
896 /* return to user mode */
897 .byte HEX(48) // REX prefix to return to long mode
898 sysret
899 .ENDP
900
901
902 PUBLIC KiSystemCallEntry32
903 KiSystemCallEntry32:
904 swapgs
905 int 3
906
907
908 PUBLIC KiZwSystemService
909 FUNC KiZwSystemService
910 push rbp
911 .pushreg rbp
912 sub rsp, KTRAP_FRAME_LENGTH
913 .allocstack KTRAP_FRAME_LENGTH
914 mov [rsp + KTRAP_FRAME_Rsi], rsi
915 .savereg rsi, KTRAP_FRAME_Rsi
916 mov [rsp + KTRAP_FRAME_Rdi], rdi
917 .savereg rdi, KTRAP_FRAME_Rdi
918 mov rbp, rsp
919 .setframe rbp, 0
920 .endprolog
921
922 /* Get current thread */
923 mov r11, gs:[PcCurrentThread]
924
925 /* Save the old trap frame in TrapFrame.Rdx */
926 mov rdi, [r11 + KTHREAD_TrapFrame]
927 mov [rbp + KTRAP_FRAME_Rdx], rdi
928
929 /* Set the new trap frame and previous mode */
930 mov [r11 + ThTrapFrame], rbp
931 mov byte ptr [r11 + KTHREAD_PreviousMode], 0
932
933 /* allocate space for parameters */
934 sub rsp, r10
935 and rsp, HEX(0fffffffffffffff0)
936
937 /* Save rcx */
938 mov [rbp + KTRAP_FRAME_Rcx], rcx
939
940 /* copy parameters to the new location */
941 lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
942 lea rdi, [rsp]
943 mov rcx, r10
944 shr rcx, 3
945 rep movsq
946
947 /* Restore rcx */
948 mov rcx, [rbp + KTRAP_FRAME_Rcx]
949
950 /* Call the service function */
951 call rax
952
953 /* Restore the old trap frame */
954 mov r11, gs:[PcCurrentThread]
955 mov rsi, [rsp + KTRAP_FRAME_Rdx]
956 mov [r11 + KTHREAD_TrapFrame], rsi
957
958 /* Restore rdi and rsi */
959 mov rsi, [rbp + KTRAP_FRAME_Rsi]
960 mov rdi, [rbp + KTRAP_FRAME_Rdi]
961
962 /* Cleanup the stack and return */
963 lea rsp, [rbp + KTRAP_FRAME_LENGTH]
964 pop rbp
965 ret
966
967 ENDFUNC
968
969
970 KiExitToUserApc:
971 int 3
972
973 /*!
974 * VOID
975 * DECLSPEC_NORETURN
976 * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
977 */
978 PUBLIC KiServiceExit
979 KiServiceExit:
980 mov [rcx + KTRAP_FRAME_Rax], rdx
981 mov rbp, rcx
982 mov rsp, rcx
983
984 /* Return */
985 //ExitTrap TF_SAVE_ALL
986
987 /*!
988 * VOID
989 * DECLSPEC_NORETURN
990 * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
991 */
992 PUBLIC KiServiceExit2
993 .PROC KiServiceExit2
994 .ENDPROLOG
995
996 mov rbp, rcx
997 mov rsp, rcx
998
999 /* Return */
1000 ExitTrap TF_SAVE_ALL
1001 .ENDP
1002
1003 PUBLIC KiInitializeSegments
1004 KiInitializeSegments:
1005 mov ax, KGDT64_R3_DATA or RPL_MASK
1006 mov gs, ax
1007 swapgs
1008 mov gs, ax
1009 ret
1010
1011
1012 #ifdef _MSC_VER
1013 #undef lgdt
1014 #undef lidt
1015
1016 //void __lgdt(void *Source);
1017 PUBLIC __lgdt
1018 __lgdt:
1019 lgdt fword ptr [rcx]
1020 ret
1021
1022 //void __sgdt(void *Destination);
1023 PUBLIC __sgdt
1024 __sgdt:
1025 sgdt fword ptr [rcx]
1026 ret
1027
1028 // void __lldt(unsigned short Value)
1029 PUBLIC __lldt
1030 __lldt:
1031 lldt cx
1032 ret
1033
1034 //void __sldt(void *Destination);
1035 PUBLIC __sldt
1036 __sldt:
1037 sldt word ptr [rcx]
1038 ret
1039
1040 //void __ltr(unsigned short Source);
1041 PUBLIC __ltr
1042 __ltr:
1043 ltr cx
1044 ret
1045
1046 //void __str(unsigned short *Destination);
1047 PUBLIC __str
1048 __str:
1049 str word ptr [rcx]
1050 ret
1051
1052 PUBLIC __swapgs
1053 __swapgs:
1054 swapgs
1055 ret
1056
1057 #endif
1058
1059 END