X-Git-Url: https://git.reactos.org/?p=reactos.git;a=blobdiff_plain;f=ntoskrnl%2Finclude%2Finternal%2Fi386%2Fke.h;h=10eb6eb1bed9f0ba5faf6f00915641cf142d939d;hp=107e099b1a9f37b95bab142591662f4afc796479;hb=71fefa32db013317842fd9224b63bcf5f72f2e32;hpb=18ac12569f32eb04c6e96545f67cc583b1c6bcaf diff --git a/ntoskrnl/include/internal/i386/ke.h b/ntoskrnl/include/internal/i386/ke.h index 107e099b1a9..10eb6eb1bed 100644 --- a/ntoskrnl/include/internal/i386/ke.h +++ b/ntoskrnl/include/internal/i386/ke.h @@ -137,9 +137,8 @@ typedef union _KTRAP_EXIT_SKIP_BITS IN ULONG Flags) \ { \ /* Not yet handled */ \ - UNIMPLEMENTED; \ - while (TRUE); \ - return TRUE; \ + UNIMPLEMENTED_DBGBREAK(); \ + return FALSE; \ } C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA)); @@ -164,6 +163,18 @@ typedef struct _KV8086_STACK_FRAME KV86_FRAME V86Frame; } KV8086_STACK_FRAME, *PKV8086_STACK_FRAME; +// +// Large Pages Support +// +typedef struct _LARGE_IDENTITY_MAP +{ + PHARDWARE_PTE TopLevelDirectory; + ULONG Cr3; + ULONG_PTR StartAddress; + ULONG PagesCount; + PVOID PagesList[30]; +} LARGE_IDENTITY_MAP, *PLARGE_IDENTITY_MAP; + /* Diable interrupts and return whether they were enabled before */ FORCEINLINE BOOLEAN @@ -243,8 +254,8 @@ FORCEINLINE VOID KeInvalidateTlbEntry(IN PVOID Address) { - /* HACK: Flush the entire TLB */ - __writecr3(__readcr3()); + /* Invalidate the TLB entry for this address */ + __invlpg(Address); } FORCEINLINE @@ -255,6 +266,19 @@ KeFlushProcessTb(VOID) __writecr3(__readcr3()); } +FORCEINLINE +VOID +KeSweepICache(IN PVOID BaseAddress, + IN SIZE_T FlushSize) +{ + // + // Always sweep the whole cache + // + UNREFERENCED_PARAMETER(BaseAddress); + UNREFERENCED_PARAMETER(FlushSize); + __wbinvd(); +} + FORCEINLINE PRKTHREAD KeGetCurrentThread(VOID) @@ -280,6 +304,24 @@ KiRundownThread(IN PKTHREAD Thread) #endif } +FORCEINLINE +VOID +Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress) +{ + GdtEntry->BaseLow = (USHORT)((ULONG_PTR)BaseAddress & 0xFFFF); + GdtEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)BaseAddress >> 16); + GdtEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)BaseAddress >> 24); +} + +FORCEINLINE +VOID +KiSetTebBase(PKPCR Pcr, PVOID TebAddress) +{ + Pcr->NtTib.Self = TebAddress; + Ke386SetGdtEntryBase(&Pcr->GDT[KGDT_R3_TEB / sizeof(KGDTENTRY)], TebAddress); +} + +INIT_FUNCTION VOID FASTCALL Ki386InitializeTss( @@ -288,30 +330,36 @@ Ki386InitializeTss( IN PKGDTENTRY Gdt ); +INIT_FUNCTION VOID NTAPI KiSetCR0Bits(VOID); +INIT_FUNCTION VOID NTAPI KiGetCacheInformation(VOID); +INIT_FUNCTION BOOLEAN NTAPI KiIsNpxPresent( VOID ); +INIT_FUNCTION BOOLEAN NTAPI KiIsNpxErrataPresent( VOID ); +INIT_FUNCTION VOID NTAPI KiSetProcessorType(VOID); +INIT_FUNCTION ULONG NTAPI KiGetFeatureBits(VOID); @@ -346,60 +394,98 @@ Ki386SetupAndExitToV86Mode( OUT PTEB VdmTeb ); +INIT_FUNCTION VOID NTAPI KeI386VdmInitialize( VOID ); +INIT_FUNCTION ULONG_PTR NTAPI Ki386EnableGlobalPage( - IN volatile ULONG_PTR Context + IN ULONG_PTR Context +); + +INIT_FUNCTION +ULONG_PTR +NTAPI +Ki386EnableTargetLargePage( + IN ULONG_PTR Context +); + +BOOLEAN +NTAPI +Ki386CreateIdentityMap( + IN PLARGE_IDENTITY_MAP IdentityMap, + IN PVOID StartPtr, + IN ULONG Length +); + +VOID +NTAPI +Ki386FreeIdentityMap( + IN PLARGE_IDENTITY_MAP IdentityMap ); +VOID +NTAPI +Ki386EnableCurrentLargePage( + IN ULONG_PTR StartAddress, + IN ULONG Cr3 +); + +INIT_FUNCTION VOID NTAPI KiI386PentiumLockErrataFixup( VOID ); +INIT_FUNCTION VOID NTAPI KiInitializePAT( VOID ); +INIT_FUNCTION VOID NTAPI KiInitializeMTRR( IN BOOLEAN FinalCpu ); +INIT_FUNCTION VOID NTAPI KiAmdK6InitializeMTRR( VOID ); +INIT_FUNCTION VOID NTAPI KiRestoreFastSyscallReturnState( VOID ); +INIT_FUNCTION ULONG_PTR NTAPI Ki386EnableDE( IN ULONG_PTR Context ); +INIT_FUNCTION ULONG_PTR NTAPI Ki386EnableFxsr( IN ULONG_PTR Context ); +INIT_FUNCTION ULONG_PTR NTAPI Ki386EnableXMMIExceptions( @@ -412,6 +498,12 @@ VdmDispatchBop( IN PKTRAP_FRAME TrapFrame ); +BOOLEAN +NTAPI +VdmDispatchPageFault( + _In_ PKTRAP_FRAME TrapFrame +); + BOOLEAN FASTCALL KiVdmOpcodePrefix( @@ -449,6 +541,7 @@ VOID NTAPI KiDispatchExceptionFromTrapFrame( IN NTSTATUS Code, + IN ULONG Flags, IN ULONG_PTR Address, IN ULONG ParameterCount, IN ULONG_PTR Parameter1, @@ -457,6 +550,12 @@ KiDispatchExceptionFromTrapFrame( IN PKTRAP_FRAME TrapFrame ); +NTSTATUS +NTAPI +KiConvertToGuiThread( + VOID +); + // // Global x86 only Kernel data // @@ -473,7 +572,6 @@ extern ULONG KeI386FxsrPresent; extern ULONG KiMXCsrMask; extern ULONG KeI386CpuType; extern ULONG KeI386CpuStep; -extern ULONG Ke386CacheAlignment; extern ULONG KiFastSystemCallDisable; extern UCHAR KiDebugRegisterTrapOffsets[9]; extern UCHAR KiDebugRegisterContextOffsets[9]; @@ -485,7 +583,6 @@ extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID); extern VOID NTAPI ExpInterlockedPopEntrySListResume(VOID); extern VOID __cdecl CopyParams(VOID); extern VOID __cdecl ReadBatch(VOID); -extern VOID __cdecl FrRestore(VOID); extern CHAR KiSystemCallExitBranch[]; extern CHAR KiSystemCallExit[]; extern CHAR KiSystemCallExit2[]; @@ -498,10 +595,11 @@ extern CHAR KiSystemCallExit2[]; // // Returns a thread's FPU save area // -PFX_SAVE_AREA FORCEINLINE +PFX_SAVE_AREA KiGetThreadNpxArea(IN PKTHREAD Thread) { + ASSERT((ULONG_PTR)Thread->InitialStack % 16 == 0); return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA)); } @@ -539,28 +637,6 @@ Ke386SanitizeFlags(IN ULONG Eflags, (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE))); } -// -// Gets a DR register from a CONTEXT structure -// -FORCEINLINE -PVOID -KiDrFromContext(IN ULONG Dr, - IN PCONTEXT Context) -{ - return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]); -} - -// -// Gets a DR register from a KTRAP_FRAME structure -// -FORCEINLINE -PVOID* -KiDrFromTrapFrame(IN ULONG Dr, - IN PKTRAP_FRAME TrapFrame) -{ - return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]); -} - // // Sanitizes a Debug Register // @@ -581,38 +657,38 @@ Ke386SanitizeDr(IN PVOID DrAddress, // // Exception with no arguments // -VOID FORCEINLINE DECLSPEC_NORETURN +VOID KiDispatchException0Args(IN NTSTATUS Code, IN ULONG_PTR Address, IN PKTRAP_FRAME TrapFrame) { /* Helper for exceptions with no arguments */ - KiDispatchExceptionFromTrapFrame(Code, Address, 0, 0, 0, 0, TrapFrame); + KiDispatchExceptionFromTrapFrame(Code, 0, Address, 0, 0, 0, 0, TrapFrame); } // // Exception with one argument // -VOID FORCEINLINE DECLSPEC_NORETURN +VOID KiDispatchException1Args(IN NTSTATUS Code, IN ULONG_PTR Address, IN ULONG P1, IN PKTRAP_FRAME TrapFrame) { /* Helper for exceptions with no arguments */ - KiDispatchExceptionFromTrapFrame(Code, Address, 1, P1, 0, 0, TrapFrame); + KiDispatchExceptionFromTrapFrame(Code, 0, Address, 1, P1, 0, 0, TrapFrame); } // // Exception with two arguments // -VOID FORCEINLINE DECLSPEC_NORETURN +VOID KiDispatchException2Args(IN NTSTATUS Code, IN ULONG_PTR Address, IN ULONG P1, @@ -620,7 +696,7 @@ KiDispatchException2Args(IN NTSTATUS Code, IN PKTRAP_FRAME TrapFrame) { /* Helper for exceptions with no arguments */ - KiDispatchExceptionFromTrapFrame(Code, Address, 2, P1, P2, 0, TrapFrame); + KiDispatchExceptionFromTrapFrame(Code, 0, Address, 2, P1, P2, 0, TrapFrame); } // @@ -643,8 +719,8 @@ KiDispatchException2Args(IN NTSTATUS Code, * */ #ifdef __GNUC__ -NTSTATUS FORCEINLINE +NTSTATUS KiSystemCallTrampoline(IN PVOID Handler, IN PVOID Arguments, IN ULONG StackBytes) @@ -653,13 +729,13 @@ KiSystemCallTrampoline(IN PVOID Handler, __asm__ __volatile__ ( - "subl %1, %%esp\n" - "movl %%esp, %%edi\n" - "movl %2, %%esi\n" - "shrl $2, %1\n" - "rep movsd\n" - "call *%3\n" - "movl %%eax, %0\n" + "subl %1, %%esp\n\t" + "movl %%esp, %%edi\n\t" + "movl %2, %%esi\n\t" + "shrl $2, %1\n\t" + "rep movsd\n\t" + "call *%3\n\t" + "movl %%eax, %0" : "=r"(Result) : "c"(StackBytes), "d"(Arguments), @@ -669,8 +745,8 @@ KiSystemCallTrampoline(IN PVOID Handler, return Result; } #elif defined(_MSC_VER) -NTSTATUS FORCEINLINE +NTSTATUS KiSystemCallTrampoline(IN PVOID Handler, IN PVOID Arguments, IN ULONG StackBytes) @@ -696,8 +772,8 @@ KiSystemCallTrampoline(IN PVOID Handler, // // Checks for pending APCs // -VOID FORCEINLINE +VOID KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) { PKTHREAD Thread; @@ -730,77 +806,32 @@ KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame) } } -// -// Converts a base thread to a GUI thread -// -#ifdef __GNUC__ -NTSTATUS -FORCEINLINE -KiConvertToGuiThread(VOID) -{ - NTSTATUS Result; - PVOID StackFrame; - - /* - * Converting to a GUI thread safely updates ESP in-place as well as the - * current Thread->TrapFrame and EBP when KeSwitchKernelStack is called. - * - * However, PsConvertToGuiThread "helpfully" restores EBP to the original - * caller's value, since it is considered a nonvolatile register. As such, - * as soon as we're back after the conversion and we try to store the result - * which will probably be in some stack variable (EBP-based), we'll crash as - * we are touching the de-allocated non-expanded stack. - * - * Thus we need a way to update our EBP before EBP is touched, and the only - * way to guarantee this is to do the call itself in assembly, use the EAX - * register to store the result, fixup EBP, and then let the C code continue - * on its merry way. - * - */ - __asm__ __volatile__ - ( - "movl %%ebp, %1\n\t" - "subl %%esp, %1\n\t" - "call _PsConvertToGuiThread@0\n\t" - "addl %%esp, %1\n\t" - "movl %1, %%ebp" - : "=a"(Result), "=r"(StackFrame) - : - : "%esp", "%ecx", "%edx", "memory" - ); - return Result; -} -#elif defined(_MSC_VER) -NTSTATUS -NTAPI -KiConvertToGuiThread(VOID); -#else -#error Unknown Compiler -#endif - // // Switches from boot loader to initial kernel stack // -VOID +INIT_FUNCTION FORCEINLINE +VOID KiSwitchToBootStack(IN ULONG_PTR InitialStack) { + INIT_FUNCTION VOID NTAPI KiSystemStartupBootStack(VOID); + /* We have to switch to a new stack before continuing kernel initialization */ #ifdef __GNUC__ __asm__ ( - "movl %0, %%esp\n" - "subl %1, %%esp\n" - "pushl %2\n" - "jmp _KiSystemStartupBootStack@0\n" + "movl %0, %%esp\n\t" + "subl %1, %%esp\n\t" + "pushl %2\n\t" + "jmp _KiSystemStartupBootStack@0" : : "c"(InitialStack), "i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH), - "i"(CR0_EM | CR0_TS | CR0_MP) + "i"(CR0_EM | CR0_TS | CR0_MP), + "p"(KiSystemStartupBootStack) : "%esp" ); #elif defined(_MSC_VER) - VOID NTAPI KiSystemStartupBootStack(VOID); __asm { mov esp, InitialStack @@ -816,15 +847,15 @@ KiSwitchToBootStack(IN ULONG_PTR InitialStack) // // Emits the iret instruction for C code // +FORCEINLINE DECLSPEC_NORETURN VOID -FORCEINLINE KiIret(VOID) { #if defined(__GNUC__) __asm__ __volatile__ ( - "iret\n" + "iret" ); #elif defined(_MSC_VER) __asm @@ -841,8 +872,8 @@ KiIret(VOID) // Normally this is done by the HAL, but on x86 as an optimization, the kernel // initiates the end by calling back into the HAL and exiting the trap here. // -VOID FORCEINLINE +VOID KiEndInterrupt(IN KIRQL Irql, IN PKTRAP_FRAME TrapFrame) { @@ -857,14 +888,14 @@ KiEndInterrupt(IN KIRQL Irql, // // PERF Code // -VOID FORCEINLINE +VOID Ki386PerfEnd(VOID) { extern ULONGLONG BootCyclesEnd, BootCycles; BootCyclesEnd = __rdtsc(); - DbgPrint("Boot took %I64d cycles!\n", BootCyclesEnd - BootCycles); - DbgPrint("Interrupts: %d System Calls: %d Context Switches: %d\n", + DbgPrint("Boot took %I64u cycles!\n", BootCyclesEnd - BootCycles); + DbgPrint("Interrupts: %u System Calls: %u Context Switches: %u\n", KeGetCurrentPrcb()->InterruptCount, KeGetCurrentPrcb()->KeSystemCalls, KeGetContextSwitches(KeGetCurrentPrcb()));