#include <trapamd64.inc>
EXTERN KiDispatchException:PROC
-EXTERN FrLdrDbgPrint:DWORD
EXTERN KeBugCheckWithTf:PROC
EXTERN MmAccessFault:PROC
EXTERN KiSystemFatalException:PROC
EXTERN KiGeneralProtectionFaultHandler:PROC
EXTERN KiXmmExceptionHandler:PROC
EXTERN KiDeliverApc:PROC
-EXTERN KiDispatchInterrupt:PROC
+EXTERN KiDpcInterruptHandler:PROC
+EXTERN PsConvertToGuiThread:PROC
+EXTERN MmCreateKernelStack:PROC
+EXTERN MmDeleteKernelStack:PROC
-/* GLOBALS *******************************************************************/
-
-.data
-
-PUBLIC MsgUnimplemented
-MsgUnimplemented:
-.asciz "WARNING: %s at %s:%d is UNIMPLEMENTED!\n"
+EXTERN KdSetOwedBreakpoints:PROC
/* Helper Macros *************************************************************/
ALIGN 8
-PUBLIC InterruptDispatchTable
-InterruptDispatchTable:
+MACRO(UnexpectedVectorStub, Vector)
+ /* This nop is to make the relative jmp address 4 bytes aligned and to
+ make the whole code 8 bytes long */
+ nop
+ /* This is a push instruction with 8bit operand. Since the instruction
+ sign extends the value to 32 bits, we need to offset it */
+PUBLIC KxUnexpectedInterrupt&Vector
+KxUnexpectedInterrupt&Vector:
+ push (Vector - 128)
+ jmp KiUnexpectedInterrupt
+ENDM
+
+PUBLIC KiUnexpectedRange
+KiUnexpectedRange:
Vector = 0
REPEAT 256
- push Vector
- jmp KiUnexpectedInterrupt
- ALIGN 8
+ UnexpectedVectorStub %Vector
Vector = Vector+1
ENDR
+PUBLIC KiUnexpectedRangeEnd
+KiUnexpectedRangeEnd:
+
+PUBLIC KiInterruptDispatchTemplate
+KiInterruptDispatchTemplate:
+ /* This instruction pushes the return address on the stack, which is the
+ address of the interrupt object's DispatchCode member, then jumps
+ to the address stored in the interrupt object's DispatchAddress member */
+ call qword ptr KiInterruptDispatchTemplate[rip - KINTERRUPT_DispatchCode + KINTERRUPT_DispatchAddress]
+
// rbp = TrapFrame, eax = ExceptionCode, edx = NumParams, r9,r10,r11 = params
-.PROC InternalDispatchException
+FUNC InternalDispatchException
/* Allocate stack space for EXCEPTION_RECORD and KEXCEPTION_FRAME */
sub rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
add rsp, EXCEPTION_RECORD_LENGTH + KEXCEPTION_FRAME_LENGTH
ret
-.ENDP
+ENDFUNC
-/* SOFTWARE INTERRUPT SERVICES ***********************************************/
+/* CPU EXCEPTION HANDLERS ****************************************************/
PUBLIC KiDivideErrorFault
FUNC KiDivideErrorFault
/* Push pseudo error code */
EnterTrap TF_SAVE_ALL
+ /* Check if the frame was from kernelmode */
+ test word ptr [rbp + KTRAP_FRAME_SegCs], 3
+ jz KiBreakpointTrapKMode
+
+ /* Enable interrupts for user-mode */
+ sti
+
+KiBreakpointTrapKMode:
/* Dispatch the exception */
- DispatchException STATUS_BREAKPOINT, 3, 0, 0, 0
+ DispatchException STATUS_BREAKPOINT, 3, BREAKPOINT_BREAK, 0, 0
/* Return */
ExitTrap TF_SAVE_ALL
PUBLIC KiBoundFault
FUNC KiBoundFault
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Check if the frame was from kernelmode */
PUBLIC KiInvalidOpcodeFault
FUNC KiInvalidOpcodeFault
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Enable interrupts */
/* Kernel mode fault */
/* Dispatch the exception */
- DispatchException STATUS_ILLEGAL_INSTRUCTION, 3, 0, 0, 0
+ DispatchException STATUS_ILLEGAL_INSTRUCTION, 0, 0, 0, 0
/* Return */
ExitTrap TF_SAVE_ALL
PUBLIC KiNpxNotAvailableFault
FUNC KiNpxNotAvailableFault
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Call the C handler */
PUBLIC KiDoubleFaultAbort
FUNC KiDoubleFaultAbort
- /* No error code */
- EnterTrap TF_SAVE_ALL
+
+ /* Hack for VBox, which "forgets" to push an error code on the stack! */
+ and rsp, HEX(FFFFFFFFFFFFFFF0)
+
+ /* A zero error code is pushed */
+ EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
+
+ int 3
/* Bugcheck */
Fatal 8 // EXCEPTION_DOUBLE_FAULT
PUBLIC KiNpxSegmentOverrunAbort
FUNC KiNpxSegmentOverrunAbort
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Bugcheck */
Fatal EXCEPTION_NPX_OVERRUN
+
jmp $
ENDFUNC
test eax, eax
jge KiGpfExit
- /* Dispatch the exception */
- DispatchException eax, 3, 0, 0, 0
+ /* Check for access violation */
+ cmp eax, STATUS_ACCESS_VIOLATION
+ je DispatchAccessViolation
-KiGpfFatal:
+ /* Dispatch privileged instruction fault */
+ DispatchException eax, 0, 0, 0, 0
+ jmp KiGpfExit
- /* Bugcheck */
- mov ecx, UNEXPECTED_KERNEL_MODE_TRAP
- mov rdx, HEX(000D) // EXCEPTION_GP_FAULT
- xor r8, r8
- mov r9, [rbp + KTRAP_FRAME_ErrorCode] // error code
- sub rsp, 8
- mov [rsp + KTRAP_FRAME_P5+8], rbp // trap frame
- call KeBugCheckWithTf
+DispatchAccessViolation:
+
+ /* Dispatch access violation */
+ DispatchException eax, 2, 0, -1, 0
KiGpfExit:
- /* Return */
/* Return */
ExitTrap TF_SAVE_ALL
ENDFUNC
mov rdx, cr2
mov [rbp + KTRAP_FRAME_FaultAddress], rdx
+ /* If interrupts are off, do not enable them */
+ test dword ptr [rbp + KTRAP_FRAME_EFlags], EFLAGS_IF_MASK
+ jz IntsDisabled
+
+ /* Enable interrupts for the page fault handler */
+ sti
+
+IntsDisabled:
+
/* Call page fault handler */
- mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // StoreInstruction
- and ecx, 1
+ mov ecx, [rbp + KTRAP_FRAME_ErrorCode] // FaultCode
// rdx == Address
mov r8b, [rbp + KTRAP_FRAME_SegCs] // Mode
and r8b, 1
/* Check for success */
test eax, eax
- jge PageFaultReturn
+ jl PageFaultError
+
+ /* Check whether the kernel debugger has owed breakpoints to be inserted */
+ call KdSetOwedBreakpoints
+ /* We succeeded, return */
+ jmp PageFaultReturn
+
+PageFaultError:
/* Set parameter 1 to error code */
mov r9d, [rbp + KTRAP_FRAME_ErrorCode]
- /* Set parameter2 to faulting address */
+ /* Set parameter 2 to faulting address */
mov r10, cr2 // Param2 = faulting address
cmp eax, STATUS_ACCESS_VIOLATION
je SpecialCode
InPageException:
-
/* Dispatch in-page exception */
mov r11d, eax // Param3 = Status
mov eax, STATUS_IN_PAGE_ERROR // ExceptionCode
call InternalDispatchException
PageFaultReturn:
+
+ /* Disable interrupts for the return */
+ cli
+
/* Return */
- ExitTrap TF_SAVE_ALL
+ ExitTrap (TF_SAVE_ALL or TF_CHECKUSERAPC)
ENDFUNC
PUBLIC KiFloatingErrorFault
FUNC KiFloatingErrorFault
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
UNIMPLEMENTED KiFloatingErrorFault
PUBLIC KiMcheckAbort
FUNC KiMcheckAbort
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Bugcheck */
PUBLIC KiXmmException
FUNC KiXmmException
- /* No error code */
+ /* Push pseudo error code */
EnterTrap TF_SAVE_ALL
/* Call the C handler */
jge KiXmmExit
/* Dispatch the exception */
- DispatchException eax, 3, 0, 0, 0
+ DispatchException eax, 2, 0, [rbp+KTRAP_FRAME_MxCsr], 0
+
+ // FIXME: STATUS_FLOAT_MULTIPLE_TRAPS / STATUS_FLOAT_MULTIPLE_FAULTS
KiXmmExit:
/* Return */
ENDFUNC
+/* SOFTWARE INTERRUPT SERVICES ***********************************************/
+
PUBLIC KiRaiseAssertion
FUNC KiRaiseAssertion
/* We have an error code */
- EnterTrap (TF_HAS_ERROR_CODE OR TF_SAVE_ALL)
+ EnterTrap (TF_SAVE_ALL)
/* Decrement RIP to point to the INT2C instruction (2 bytes, not 1 like INT3) */
sub qword ptr [rbp + KTRAP_FRAME_Rip], 2
PUBLIC KiDebugServiceTrap
-.PROC KiDebugServiceTrap
+FUNC KiDebugServiceTrap
/* No error code */
EnterTrap TF_SAVE_ALL
/* Return */
ExitTrap TF_SAVE_ALL
-.ENDP
+ENDFUNC
PUBLIC KiApcInterrupt
.PROC KiApcInterrupt
/* No error code */
- EnterTrap (TF_VOLATILES or TF_IRQL)
+ EnterTrap (TF_SAVE_ALL or TF_IRQL)
/* Raise to APC_LEVEL */
mov rax, APC_LEVEL
mov cl, [rbp + KTRAP_FRAME_SegCs] // ProcessorMode
and cl, 1
mov rdx, 0 // ExceptionFrame
- mov r8, rdx // TrapFrame
+ mov r8, rbp // TrapFrame
call KiDeliverApc
/* Disable interrupts */
cli
+ /* Lower IRQL back to PASSIVE */
+ mov rax, PASSIVE_LEVEL
+ mov cr8, rax
+
/* Return */
- ExitTrap (TF_VOLATILES or TF_IRQL)
+ ExitTrap (TF_SAVE_ALL or TF_IRQL)
.ENDP
+/*
+ * VOID
+ * KiRetireDpcList(
+ * PKPRCB Prcb);
+ */
+EXTERN KiRetireDpcList:PROC
+
+/*
+ * VOID
+ * KiRetireDpcListInDpcStack(
+ * PKPRCB Prcb,
+ * PVOID DpcStack);
+ */
+PUBLIC KiRetireDpcListInDpcStack
+.PROC KiRetireDpcListInDpcStack
+ push rbp
+ .pushreg rbp
+ mov rbp, rsp
+ .setframe rbp, 0
+ .endprolog
+
+ /* Switch to the DpcStack */
+ mov rsp, rdx
+
+ /* The stack is 16 byte aligned, allocate 32 bytes home space */
+ sub rsp, 32
+
+ /* Call KiRetireDpcList on the given stack */
+ call KiRetireDpcList
+
+ /* Restore stack, cleanup and return */
+ mov rsp, rbp
+ pop rbp
+ ret
+.ENDP
PUBLIC KiDpcInterrupt
.PROC KiDpcInterrupt
/* No error code */
- EnterTrap (TF_VOLATILES or TF_IRQL)
-
- /* Raise to DISPATCH_LEVEL */
- mov rax, DISPATCH_LEVEL
- mov cr8, rax
-
- /* End the interrupt */
- mov dword ptr [APIC_EOI], 0
+ EnterTrap (TF_SAVE_ALL or TF_IRQL)
/* Call the worker routine */
- sti
- call KiDispatchInterrupt
- cli
+ call KiDpcInterruptHandler
- /* Return */
- ExitTrap (TF_VOLATILES or TF_IRQL)
+ /* Return, but don't send an EOI! */
+ ExitTrap (TF_SAVE_ALL or TF_IRQL)
.ENDP
PUBLIC KiIpiInterrupt
.PROC KiIpiInterrupt
/* No error code */
- EnterTrap (TF_VOLATILES or TF_IRQL)
+ EnterTrap (TF_SAVE_ALL or TF_IRQL)
/* Raise to IPI_LEVEL */
mov rax, IPI_LEVEL
int 3
/* Return */
- ExitTrap (TF_VOLATILES or TF_IRQL)
+ ExitTrap (TF_SAVE_ALL or TF_IRQL)
.ENDP
ExitTrap TF_SAVE_ALL
ENDFUNC
+PUBLIC KiInterruptDispatch
+FUNC KiInterruptDispatch
+ /* The error code is a pointer to the interrupt object's code */
+ EnterTrap (TF_HAS_ERROR_CODE or TF_SAVE_ALL or TF_IRQL)
+
+ /* Increase interrupt count */
+ inc dword ptr gs:[PcInterruptCount];
+
+ /* Save rbx and rsi in the trap frame */
+ mov [rbp + KTRAP_FRAME_Rbx], rbx
+ mov [rbp + KTRAP_FRAME_Rsi], rsi
+
+ /* Load the address of the dispatch code into rbx */
+ mov rbx, [rbp + KTRAP_FRAME_ErrorCode]
+
+ /* Substract offset of the DispatchCode member plus 6 for the call instruction */
+ sub rbx, KINTERRUPT_DispatchCode + 6
+
+ /* Save the address of the InterruptListEntry in rsi */
+ lea rsi, [rbx + KINTERRUPT_InterruptListEntry]
+
+.DoDispatchInterrupt:
+ /* Raise IRQL to SynchronizeIrql */
+ movzx rax, byte ptr [rbx + KINTERRUPT_SynchronizeIrql]
+ mov cr8, rax
+
+#ifdef CONFIG_SMP
+ /* Acquire interrupt lock */
+ mov r8, [rbx + KINTERRUPT_ActualLock]
+
+ //KxAcquireSpinLock(Interrupt->ActualLock);
+#endif
+
+ /* Call the ISR */
+ mov rcx, rbx
+ mov rdx, [rbx + KINTERRUPT_ServiceContext]
+ call qword ptr [rbx + KINTERRUPT_ServiceRoutine]
+
+#ifdef CONFIG_SMP
+ /* Release interrupt lock */
+ //KxReleaseSpinLock(Interrupt->ActualLock);
+#endif
+
+ /* Go back to old irql */
+ movzx rax, byte ptr [rbp + KTRAP_FRAME_PreviousIrql]
+ mov cr8, rax
+
+ /* Check for chained interrupts */
+ mov rax, [rbx + KINTERRUPT_InterruptListEntry]
+ cmp rax, rsi
+ je .Done
+
+ /* Load the next interrupt object into rbx and repeat */
+ lea rbx, [rax - KINTERRUPT_InterruptListEntry]
+ jmp .DoDispatchInterrupt
+
+.Done:
+ /* Restore rbx and rsi */
+ mov rbx, [rbp + KTRAP_FRAME_Rbx]
+ mov rsi, [rbp + KTRAP_FRAME_Rsi]
+
+ /* Return */
+ ExitTrap (TF_SAVE_ALL or TF_SEND_EOI)
+ENDFUNC
+
+EXTERN KiSystemCallHandler:PROC
+
+/*! \name KiSystemCallEntry64
+ *
+ * \brief This is the entrypoint for syscalls from 64bit user mode
+ *
+ * \param rax - The system call number
+ * \param rcx - User mode return address, set by the syscall instruction
+ * \param rdx,r8,r9 - Parameters 2-4 to the service function
+ * \param r10 - Parameter 1 to the service function
+ * \param r11 - RFLAGS saved by the syscall instruction
+ *--*/
+PUBLIC KiSystemCallEntry64
+.PROC KiSystemCallEntry64
+ /* The unwind info pretends we have a machine frame */
+ .PUSHFRAME
+ .ALLOCSTACK (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE - MachineFrameLength)
+ .SAVEREG rbp, MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp
+ .ENDPROLOG
+
+ /* Swap gs to kernel, so we can access the PCR */
+ swapgs
+
+ /* Save the user mode rsp in the PCR */
+ mov gs:[PcUserRsp], rsp
+
+ /* Get the kernel stack from the PCR */
+ mov rsp, gs:[PcRspBase]
+
+ /* Allocate a TRAP_FRAME and space for parameters */
+ sub rsp, (KTRAP_FRAME_LENGTH + MAX_SYSCALL_PARAM_SIZE)
+
+ /* Save volatile registers and rbp in the trap frame */
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip], rcx
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rdx
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], r8
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], r9
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], r10
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags], r11
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
+
+ /* Store user stack pointer in the trap frame */
+ mov rax, gs:[PcUserRsp]
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp], rax
+
+ /* Set sane segments */
+ mov ax, (KGDT64_R3_DATA or RPL_MASK)
+ mov ds, ax
+ mov es, ax
+
+ /* Save MCXSR and set kernel value */
+ stmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
+ ldmxcsr gs:[PcMxCsr]
+
+ /* Get the current thread and the trap frame */
+ mov rax, gs:[PcCurrentThread]
+ mov rcx, [rax + ThTrapFrame]
+
+ /* Save the old trap frame */
+ lea rdx, [rsp + MAX_SYSCALL_PARAM_SIZE]
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame], rcx
+ mov [rax + ThTrapFrame], rdx
+
+#if DBG
+ /* Check IRQL */
+ mov rax, cr8
+ test eax, eax
+ jz KiSystemCall64Again
+ int HEX(2C)
+#endif
+
+GLOBAL_LABEL KiSystemCall64Again
+
+ /* Call the C-handler (will enable interrupts) */
+ call KiSystemCallHandler
+
+ /* The return value from KiSystemCallHandler is the address of the Nt-function */
+ mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx]
+ mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
+ mov r8, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
+ mov r9, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
+ call rax
+
+GLOBAL_LABEL KiSystemServiceExit
+
+ ASSERT_TRAP_FRAME_INTS_ENABLED rsp + MAX_SYSCALL_PARAM_SIZE
+
+ /* Check for pending user APCs */
+ mov rcx, gs:[PcCurrentThread]
+ cmp byte ptr [rcx + ThApcState + AsUserApcPending], 0
+ jz NoUserApcPending
+
+ /* Save missing regs in the trap frame */
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rax], rax
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rbp], rbp
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9], rbp
+ mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8], rax
+ mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rcx], rax
+ mov rax, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R11], rax
+ xor rax, rax
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx], rax
+ mov [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R10], rax
+ pxor xmm0, xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm0], xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm1], xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm2], xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm3], xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm4], xmm0
+ movdqa [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Xmm5], xmm0
+
+ lea rcx, [rsp + MAX_SYSCALL_PARAM_SIZE]
+ call KiInitiateUserApc
+
+NoUserApcPending:
+ /* Disable interrupts for return */
+ cli
+
+ /* Restore MCXSR */
+ ldmxcsr [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_MxCsr]
+
+ /* Restore old trap frame */
+ mov rcx, gs:[PcCurrentThread]
+ mov rdx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_TrapFrame]
+ mov [rcx + KTHREAD_TrapFrame], rdx
+
+ /* Prepare user mode return address (rcx) and eflags (r11) for sysret */
+ mov rcx, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
+ mov r11, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_EFlags]
+
+ /* Load user mode stack (It was copied to the trap frame) */
+ mov rsp, [rsp + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rsp]
+
+ /* r8 points to the user stack */
+ mov r8, rsp
+
+ /* r9 matches rbp */
+ mov r9, rbp
+
+ /* Swap gs back to user */
+ swapgs
+
+ /* Zero out volatiles */
+ pxor xmm0, xmm0
+ pxor xmm1, xmm1
+ pxor xmm2, xmm2
+ pxor xmm3, xmm3
+ pxor xmm4, xmm4
+ pxor xmm5, xmm5
+ xor rdx, rdx
+ xor r10, r10
+
+ /* return to user mode */
+ sysretq
+
+.ENDP
+
+
+/*!
+ * VOID
+ * DECLSPEC_NORETURN
+ * KiServiceExit(IN PKTRAP_FRAME TrapFrame, IN NTSTATUS Status));
+ */
+PUBLIC KiServiceExit
+.PROC KiServiceExit
+ .endprolog
+
+ lea rsp, [rcx - MAX_SYSCALL_PARAM_SIZE]
+ jmp KiSystemServiceExit
+
+.ENDP
+
+
+/*!
+ * VOID
+ * DECLSPEC_NORETURN
+ * KiServiceExit2(IN PKTRAP_FRAME TrapFrame);
+ */
+PUBLIC KiServiceExit2
+.PROC KiServiceExit2
+ .ENDPROLOG
+
+ // FIXME: this should probably also restore an exception frame
+
+ mov rsp, rcx
+.ENDP
+
+PUBLIC KiServiceExit3
+.PROC KiServiceExit3
+ .PUSHFRAME
+ .ALLOCSTACK (KTRAP_FRAME_LENGTH - MachineFrameLength)
+ .ENDPROLOG
+
+#if DBG
+ /* Get the current IRQL and compare it to the trap frame */
+ mov rax, cr8
+ cmp byte ptr [rsp + KTRAP_FRAME_PreviousIrql], al
+ je KiServiceExit2_ok1
+ int HEX(2C)
+
+KiServiceExit2_ok1:
+ /* Check if this is a user mode exit */
+ mov ah, byte ptr [rsp + KTRAP_FRAME_SegCs]
+ test ah, 1
+ jz KiServiceExit2_kernel
+
+ /* Validate that we are at PASSIVE_LEVEL */
+ test al, al
+ jz KiServiceExit2_kernel
+ int HEX(2C)
+
+KiServiceExit2_kernel:
+#endif
+
+ /* Return */
+ mov rbp, rsp
+ ExitTrap TF_SAVE_ALL
+.ENDP
+
+
+PUBLIC KiSystemCallEntry32
+KiSystemCallEntry32:
+ swapgs
+ int 3
+
+
+PUBLIC KiZwSystemService
+FUNC KiZwSystemService
+ push rbp
+ .pushreg rbp
+ sub rsp, KTRAP_FRAME_LENGTH
+ .allocstack KTRAP_FRAME_LENGTH
+ mov [rsp + KTRAP_FRAME_Rsi], rsi
+ .savereg rsi, KTRAP_FRAME_Rsi
+ mov [rsp + KTRAP_FRAME_Rdi], rdi
+ .savereg rdi, KTRAP_FRAME_Rdi
+ mov rbp, rsp
+ .setframe rbp, 0
+ .endprolog
+
+ /* Get current thread */
+ mov r11, gs:[PcCurrentThread]
+
+ /* Save PreviousMode in the trap frame */
+ mov dil, byte ptr [r11 + KTHREAD_PreviousMode]
+ mov byte ptr [rbp + KTRAP_FRAME_PreviousMode], dil
+
+ /* Save the old trap frame in TrapFrame.Rdx */
+ mov rdi, [r11 + KTHREAD_TrapFrame]
+ mov [rbp + KTRAP_FRAME_Rdx], rdi
+
+ /* Set the new trap frame and previous mode */
+ mov [r11 + ThTrapFrame], rbp
+ mov byte ptr [r11 + KTHREAD_PreviousMode], 0
+
+ /* allocate space for parameters */
+ sub rsp, r10
+ and rsp, HEX(0fffffffffffffff0)
+
+ /* Save rcx */
+ mov [rbp + KTRAP_FRAME_Rcx], rcx
+
+ /* copy parameters to the new location */
+ lea rsi, [rbp + KTRAP_FRAME_LENGTH + 16]
+ lea rdi, [rsp]
+ mov rcx, r10
+ shr rcx, 3
+ rep movsq
+
+ /* Restore rcx */
+ mov rcx, [rbp + KTRAP_FRAME_Rcx]
+
+ /* Call the service function */
+ call rax
+
+ /* Restore the old trap frame */
+ mov r11, gs:[PcCurrentThread]
+ mov rsi, [rbp + KTRAP_FRAME_Rdx]
+ mov [r11 + KTHREAD_TrapFrame], rsi
+
+ /* Restore PreviousMode from the trap frame */
+ mov dil, byte ptr [rbp + KTRAP_FRAME_PreviousMode]
+ mov byte ptr [r11 + KTHREAD_PreviousMode], dil
+
+ /* Restore rdi and rsi */
+ mov rsi, [rbp + KTRAP_FRAME_Rsi]
+ mov rdi, [rbp + KTRAP_FRAME_Rdi]
+
+ /* Cleanup the stack and return */
+ lea rsp, [rbp + KTRAP_FRAME_LENGTH]
+ pop rbp
+ ret
+
+ENDFUNC
+
+PUBLIC KiConvertToGuiThread
+FUNC KiConvertToGuiThread
+
+ sub rsp, 40
+ .allocstack 40
+ .endprolog
+
+ /* Check if we already have a large stack */
+ mov rax, gs:[PcCurrentThread]
+ cmp byte ptr [rax + KTHREAD_LargeStack], 0
+ jnz AlreadyLargeStack
+
+ // NewStack = (ULONG_PTR)MmCreateKernelStack(TRUE, 0);
+ mov cl, 1
+ xor rdx, rdx
+ call MmCreateKernelStack
+
+ /* Check for failure */
+ test rax, rax
+ jz KiConvertToGuiThreadFailed
+
+ /* OldStack = KeSwitchKernelStack((PVOID)NewStack, (PVOID)(NewStack - KERNEL_LARGE_STACK_COMMIT )); */
+ mov rcx, rax
+ mov rdx, rax
+ sub rdx, KERNEL_LARGE_STACK_COMMIT
+ call KeSwitchKernelStack
+
+ // MmDeleteKernelStack(OldStack, FALSE);
+ mov rcx, rax
+ xor rdx, rdx
+ call MmDeleteKernelStack
+
+AlreadyLargeStack:
+
+ /* Call the worker function */
+ call PsConvertToGuiThread
+
+ /* Check for failure */
+ test eax, eax
+ js KiConvertToGuiThreadFailed
+
+ /* Disable interrupts for return */
+ cli
+
+ // Restore register parameters
+ mov rcx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rip]
+ mov rdx, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_Rdx]
+ mov r8, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R8]
+ mov r9, [rsp + 48 + MAX_SYSCALL_PARAM_SIZE + KTRAP_FRAME_R9]
+
+ /* Run KiSystemCallHandler again */
+ add rsp, 48
+ jmp KiSystemCall64Again
+
+KiConvertToGuiThreadFailed:
+
+ /* Clean up the stack and return failure */
+ add rsp, 40
+ mov eax, HEX(C0000017) // STATUS_NO_MEMORY
+ ret
+
+ENDFUNC
+
+
+EXTERN KiSetTrapContextInternal:PROC
+
+/*
+ * VOID
+ * KiSetTrapContext(
+ * _Out_ PKTRAP_FRAME TrapFrame,
+ * _In_ PCONTEXT Context,
+ * _In_ KPROCESSOR_MODE RequestorMode);
+ */
+PUBLIC KiSetTrapContext
+.PROC KiSetTrapContext
+
+ /* Generate a KEXCEPTION_FRAME on the stack */
+ GENERATE_EXCEPTION_FRAME
+
+ call KiSetTrapContextInternal
+
+ /* Restore the registers from the KEXCEPTION_FRAME */
+ RESTORE_EXCEPTION_STATE
+
+ /* Return */
+ ret
+
+.ENDP
+
+
+/*
+ * VOID
+ * KiDeliverApc(
+ * _In_ KPROCESSOR_MODE DeliveryMode,
+ * _In_ PKEXCEPTION_FRAME ExceptionFrame,
+ * _In_ PKTRAP_FRAME TrapFrame);
+ *
+ */
+EXTERN KiDeliverApc:PROC
+
+/*
+ * VOID
+ * KiInitiateUserApc(
+ * _In_ PKTRAP_FRAME TrapFrame@<rcx>);
+ *
+ * This function is called to deliver user mode APCs.
+ * It clobbers all non-volatile registers, except rax.
+ */
+PUBLIC KiInitiateUserApc
+.PROC KiInitiateUserApc
+
+ /* Generate a KEXCEPTION_FRAME on the stack */
+ GENERATE_EXCEPTION_FRAME
+
+ /* Raise IRQL to APC_LEVEL */
+ mov rax, APC_LEVEL
+ mov cr8, rax
+
+ /* Get the current thread */
+ mov rbp, gs:[PcCurrentThread]
+
+ /* Save the trap frame in rsi */
+ mov rsi, rcx
+
+ /* Enable interrupts */
+ sti
+
+ /* Call the C function */
+ mov ecx, 1
+ mov rdx, rsp
+ mov r8, rsi
+ call KiDeliverApc
+
+ /* Disable interrupts again */
+ cli
+
+ /* Go back to PASSIVE_LEVEL */
+ mov rax, PASSIVE_LEVEL
+ mov cr8, rax
+
+ /* Restore the registers from the KEXCEPTION_FRAME */
+ RESTORE_EXCEPTION_STATE
+
+ /* Return */
+ ret
+
+.ENDP
+
+
+PUBLIC KiInitializeSegments
+KiInitializeSegments:
+ mov ax, KGDT64_R3_DATA or RPL_MASK
+ mov gs, ax
+ swapgs
+ mov gs, ax
+ ret
+
+/*!
+ * VOID
+ * KiSwitchKernelStackHelper(
+ * LONG_PTR StackOffset,
+ * PVOID OldStackBase);
+ */
+PUBLIC KiSwitchKernelStackHelper
+KiSwitchKernelStackHelper:
+
+ /* Pop return address from the current stack */
+ pop rax
+
+ /* Switch to new stack */
+ lea rsp, [rsp + rcx]
+
+ /* Push return address on the new stack */
+ push rax
+
+ /* Return on new stack */
+ mov rax, rdx
+ ret
+
+EXTERN KiSwitchKernelStack:PROC
+
+PUBLIC KeSwitchKernelStack
+FUNC KeSwitchKernelStack
+
+ /* Save rcx and allocate callee home space */
+ mov [rsp + P1Home], rcx
+ .savereg rcx, P1Home
+ sub rsp, 40
+ .allocstack 40
+ .endprolog
+
+ /* Call the C handler, which returns the old stack in rax */
+ call KiSwitchKernelStack
+
+ /* Restore rcx (StackBase) */
+ mov rcx, [rsp + 40 + P1Home]
+
+ /* Switch to new stack: RSP += (StackBase - OldStackBase) */
+ sub rcx, rax
+ add rsp, rcx
+
+ /* Deallocate the home frame */
+ add rsp, 40
+ ret
+
+ENDFUNC
+
+
+
+
#ifdef _MSC_VER
#undef lgdt
#undef lidt
str word ptr [rcx]
ret
+PUBLIC __swapgs
+__swapgs:
+ swapgs
+ ret
+
#endif
END
projects / reactos.git / blobdiff