From f1e71db26d48ce9f3a41dc97454b6d036e6b3375 Mon Sep 17 00:00:00 2001 From: Alex Ionescu Date: Wed, 24 Jan 2007 06:50:28 +0000 Subject: [PATCH] - Rewrite kernel timer implementation to use Windows 2003's hash-based table timer model. Extremely scalable, since each timer list has its own lock, and optimized for speed since timers are hashed with the tick count at each update (timer.c). - Implement a new timer expiration function which takes advantage of the fact that timers are now hashed with the system tickcount, and thus only parses the "hot" lists (dpc.c). - Detect timer expiration during KeUpdateSystemTime and edit the KPRCB to notify of timer expiration. This will be picked up by the timer expiration DPC (clock.s). - Add support for new timer code in dispatcher (wait.c, queue.c, ke_x.h). - Change system startup to support the new timer DPC/list (krnlinit.c, clock.c). - Remove corresponding entry from kernel fun, and update the guidance plan with recent successes and setbacks. This patch is likely the last big architectural change in the kernel except for enabling the new scheduler at a later time. svn path=/trunk/; revision=25611 --- reactos/ntoskrnl/KrnlFun.c | 18 +- reactos/ntoskrnl/include/internal/ke.h | 55 +- reactos/ntoskrnl/include/internal/ke_x.h | 1905 +++++++++++----------- reactos/ntoskrnl/ke/clock.c | 3 +- reactos/ntoskrnl/ke/dpc.c | 243 ++- reactos/ntoskrnl/ke/i386/clock.S | 86 +- reactos/ntoskrnl/ke/krnlinit.c | 9 +- reactos/ntoskrnl/ke/queue.c | 12 +- reactos/ntoskrnl/ke/timer.c | 395 +++-- reactos/ntoskrnl/ke/wait.c | 18 +- 10 files changed, 1594 insertions(+), 1150 deletions(-) diff --git a/reactos/ntoskrnl/KrnlFun.c b/reactos/ntoskrnl/KrnlFun.c index 5f27d66c918..3288f8b742e 100644 --- a/reactos/ntoskrnl/KrnlFun.c +++ b/reactos/ntoskrnl/KrnlFun.c @@ -8,9 +8,6 @@ // Do NOT ask when it will be fixed. // Failure to respect this will *ACHIEVE NOTHING*. // -// Ke2: -// - Dispatcher Rewrite (DPCs-Timers-Waits). -// // Hal: // - Use APC and DPC Interrupt Dispatchers. // - CMOS Initialization and CMOS Spinlock. @@ -26,14 +23,15 @@ // REACTOS GUIDANCE PLAN // ________________________________________________________________________________________________________ // / \ -// | OB, PS, LPC, DBGK, IO => Almost entirely fixed interaction with Ke/Ex. | | +// | OB, PS, LPC, DBGK, EX => "Code complete". No expected changes until 0.5.0 | | // | SE => Not looked at. Interaction with Ps/Io is minimal and currently hacked away. Preserve. |J| -// | EX => Needs re-visiting (in trunk). Do callbacks/push locks for interaction with Ps. |A| +// | INIT => Boot sequence still needs work in terms of interaction with Ke and CPU features. |A| // | KD/KDBG => Laptop has special version of ROS without these components. Commit in branch. |N| -// | INIT => Boot sequence still needs work in terms of interaction with Ke and CPU features. | | -// | || || || || || || || || || || || || |F| -// | \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ |E| -// | HAL => Needs APC/DPC/IRQL implementation fixed ASAP in terms of interaction with Ke. |B| +// | HAL => Needs APC/DPC/IRQL implementation fixed ASAP in terms of interaction with Ke. | | +// | || || || || || || || || || || || || | | +// | \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ |F| +// | KE => Enable new thread scheduler and ensure it works. |E| +// | KD => Implement KD64 6.0, compatible with WinDBG |B| // | FSTUB => Needs IoAssignDriveLetters fixed ASAP but not critical to Ke/Ex. Interacts with Io. | | // | || || || || || || || || || || || || |M| // | \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ |A| @@ -41,7 +39,7 @@ // | || || || || || || || || || || || || | | // | || || || || || || || || || || || || |A| // | \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ \/ |P| -// | KE => Timer Rewrite + Thread Scheduler Rewrite. |R| +// | PNPMGR => TBD |R| // | || || || || || || || || || || || || |I| // | || || || || || || || || || || || || |L| // | || || || || || || || || || || || || | | diff --git a/reactos/ntoskrnl/include/internal/ke.h b/reactos/ntoskrnl/include/internal/ke.h index 9e213f6f327..3d079faa3ea 100644 --- a/reactos/ntoskrnl/include/internal/ke.h +++ b/reactos/ntoskrnl/include/internal/ke.h @@ -55,6 +55,15 @@ typedef struct _KTIMER_TABLE_ENTRY ULARGE_INTEGER Time; } KTIMER_TABLE_ENTRY, *PKTIMER_TABLE_ENTRY; +#define MAX_TIMER_DPCS 16 + +typedef struct _DPC_QUEUE_ENTRY +{ + PKDPC Dpc; + PKDEFERRED_ROUTINE Routine; + PVOID Context; +} DPC_QUEUE_ENTRY, *PDPC_QUEUE_ENTRY; + typedef PCHAR (NTAPI *PKE_BUGCHECK_UNICODE_TO_ANSI)( IN PUNICODE_STRING Unicode, @@ -122,9 +131,8 @@ extern ULONG KiTimeLimitIsrMicroseconds; extern ULONG KiServiceLimit; extern LIST_ENTRY BugcheckCallbackListHead, BugcheckReasonCallbackListHead; extern KSPIN_LOCK BugCheckCallbackLock; -extern KDPC KiExpireTimerDpc; +extern KDPC KiTimerExpireDpc; extern KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]; -extern LIST_ENTRY KiTimerListHead; extern FAST_MUTEX KiGenericCallDpcMutex; extern LIST_ENTRY KiProfileListHead, KiProfileSourceListHead; extern KSPIN_LOCK KiProfileLock; @@ -258,6 +266,27 @@ CPUID( IN ULONG InfoType ); +BOOLEAN +FASTCALL +KiInsertTimerTable( + IN PKTIMER Timer, + IN ULONG Hand +); + +BOOLEAN +FASTCALL +KiInsertTreeTimer( + IN PKTIMER Timer, + IN LARGE_INTEGER Interval +); + +VOID +FASTCALL +KiCompleteTimer( + IN PKTIMER Timer, + IN PKSPIN_LOCK_QUEUE LockQueue +); + /* gmutex.c ********************************************************************/ VOID @@ -562,13 +591,6 @@ BOOLEAN NTAPI KeDisableThreadApcQueueing(IN PKTHREAD Thread); -BOOLEAN -NTAPI -KiInsertTimer( - PKTIMER Timer, - LARGE_INTEGER DueTime -); - VOID FASTCALL KiWaitTest( @@ -598,6 +620,21 @@ KiInsertQueue( BOOLEAN Head ); +VOID +NTAPI +KiTimerExpiration( + IN PKDPC Dpc, + IN PVOID DeferredContext, + IN PVOID SystemArgument1, + IN PVOID SystemArgument2 +); + +ULONG +NTAPI +KiComputeTimerTableIndex( + IN LONGLONG TimeValue +); + ULONG NTAPI KeSetProcess( diff --git a/reactos/ntoskrnl/include/internal/ke_x.h b/reactos/ntoskrnl/include/internal/ke_x.h index 11050ad723a..0a354218c11 100644 --- a/reactos/ntoskrnl/include/internal/ke_x.h +++ b/reactos/ntoskrnl/include/internal/ke_x.h @@ -175,1125 +175,1195 @@ Ke386SanitizeDr(IN PVOID DrAddress, } \ } +#ifndef _CONFIG_SMP // -// Satisfies the wait of any dispatcher object +// Spinlock Acquire at IRQL >= DISPATCH_LEVEL // -#define KiSatisfyObjectWait(Object, Thread) \ -{ \ - /* Special case for Mutants */ \ - if ((Object)->Header.Type == MutantObject) \ - { \ - /* Decrease the Signal State */ \ - (Object)->Header.SignalState--; \ - \ - /* Check if it's now non-signaled */ \ - if (!(Object)->Header.SignalState) \ - { \ - /* Set the Owner Thread */ \ - (Object)->OwnerThread = Thread; \ - \ - /* Disable APCs if needed */ \ - Thread->KernelApcDisable = Thread->KernelApcDisable - \ - (Object)->ApcDisable; \ - \ - /* Check if it's abandoned */ \ - if ((Object)->Abandoned) \ - { \ - /* Unabandon it */ \ - (Object)->Abandoned = FALSE; \ - \ - /* Return Status */ \ - Thread->WaitStatus = STATUS_ABANDONED; \ - } \ - \ - /* Insert it into the Mutant List */ \ - InsertHeadList(Thread->MutantListHead.Blink, \ - &(Object)->MutantListEntry); \ - } \ - } \ - else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ - EventSynchronizationObject) \ - { \ - /* Synchronization Timers and Events just get un-signaled */ \ - (Object)->Header.SignalState = 0; \ - } \ - else if ((Object)->Header.Type == SemaphoreObject) \ - { \ - /* These ones can have multiple states, so we only decrease it */ \ - (Object)->Header.SignalState--; \ - } \ +FORCEINLINE +VOID +KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +{ + /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ + UNREFERENCED_PARAMETER(SpinLock); } // -// Satisfies the wait of a mutant dispatcher object +// Spinlock Release at IRQL >= DISPATCH_LEVEL // -#define KiSatisfyMutantWait(Object, Thread) \ -{ \ - /* Decrease the Signal State */ \ - (Object)->Header.SignalState--; \ - \ - /* Check if it's now non-signaled */ \ - if (!(Object)->Header.SignalState) \ - { \ - /* Set the Owner Thread */ \ - (Object)->OwnerThread = Thread; \ - \ - /* Disable APCs if needed */ \ - Thread->KernelApcDisable = Thread->KernelApcDisable - \ - (Object)->ApcDisable; \ - \ - /* Check if it's abandoned */ \ - if ((Object)->Abandoned) \ - { \ - /* Unabandon it */ \ - (Object)->Abandoned = FALSE; \ - \ - /* Return Status */ \ - Thread->WaitStatus = STATUS_ABANDONED; \ - } \ - \ - /* Insert it into the Mutant List */ \ - InsertHeadList(Thread->MutantListHead.Blink, \ - &(Object)->MutantListEntry); \ - } \ +FORCEINLINE +VOID +KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +{ + /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ + UNREFERENCED_PARAMETER(SpinLock); } // -// Satisfies the wait of any nonmutant dispatcher object +// This routine protects against multiple CPU acquires, it's meaningless on UP. // -#define KiSatisfyNonMutantWait(Object) \ -{ \ - if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ - EventSynchronizationObject) \ - { \ - /* Synchronization Timers and Events just get un-signaled */ \ - (Object)->Header.SignalState = 0; \ - } \ - else if ((Object)->Header.Type == SemaphoreObject) \ - { \ - /* These ones can have multiple states, so we only decrease it */ \ - (Object)->Header.SignalState--; \ - } \ +VOID +FORCEINLINE +KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + UNREFERENCED_PARAMETER(Object); } // -// Recalculates the due time +// This routine protects against multiple CPU acquires, it's meaningless on UP. // -PLARGE_INTEGER +VOID FORCEINLINE -KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, - IN PLARGE_INTEGER DueTime, - IN OUT PLARGE_INTEGER NewDueTime) +KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) { - /* Don't do anything for absolute waits */ - if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime; + UNREFERENCED_PARAMETER(Object); +} - /* Otherwise, query the interrupt time and recalculate */ - NewDueTime->QuadPart = KeQueryInterruptTime(); - NewDueTime->QuadPart -= DueTime->QuadPart; - return NewDueTime; +KIRQL +FORCEINLINE +KiAcquireDispatcherLock(VOID) +{ + /* Raise to DPC level */ + return KeRaiseIrqlToDpcLevel(); +} + +VOID +FORCEINLINE +KiReleaseDispatcherLock(IN KIRQL OldIrql) +{ + /* Just exit the dispatcher */ + KiExitDispatcher(OldIrql); +} + +VOID +FORCEINLINE +KiAcquireDispatcherLockAtDpcLevel(VOID) +{ + /* This is a no-op at DPC Level for UP systems */ + return; +} + +VOID +FORCEINLINE +KiReleaseDispatcherLockFromDpcLevel(VOID) +{ + /* This is a no-op at DPC Level for UP systems */ + return; } // -// Determines wether a thread should be added to the wait list +// This routine makes the thread deferred ready on the boot CPU. // FORCEINLINE -BOOLEAN -KiCheckThreadStackSwap(IN PKTHREAD Thread, - IN KPROCESSOR_MODE WaitMode) +VOID +KiInsertDeferredReadyList(IN PKTHREAD Thread) { - /* Check the required conditions */ - if ((WaitMode != KernelMode) && - (Thread->EnableStackSwap) && - (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9))) - { - /* We are go for swap */ - return TRUE; - } - else - { - /* Don't swap the thread */ - return FALSE; - } + /* Set the thread to deferred state and boot CPU */ + Thread->State = DeferredReady; + Thread->DeferredProcessor = 0; + + /* Make the thread ready immediately */ + KiDeferredReadyThread(Thread); +} + +FORCEINLINE +VOID +KiRescheduleThread(IN BOOLEAN NewThread, + IN ULONG Cpu) +{ + /* This is meaningless on UP systems */ + UNREFERENCED_PARAMETER(NewThread); + UNREFERENCED_PARAMETER(Cpu); } // -// Adds a thread to the wait list +// This routine protects against multiple CPU acquires, it's meaningless on UP. // -#define KiAddThreadToWaitList(Thread, Swappable) \ -{ \ - /* Make sure it's swappable */ \ - if (Swappable) \ - { \ - /* Insert it into the PRCB's List */ \ - InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \ - &Thread->WaitListEntry); \ - } \ +FORCEINLINE +VOID +KiSetThreadSwapBusy(IN PKTHREAD Thread) +{ + UNREFERENCED_PARAMETER(Thread); } // -// Checks if a wait in progress should be interrupted by APCs or an alertable -// state. +// This routine protects against multiple CPU acquires, it's meaningless on UP. // FORCEINLINE -NTSTATUS -KiCheckAlertability(IN PKTHREAD Thread, - IN BOOLEAN Alertable, - IN KPROCESSOR_MODE WaitMode) +VOID +KiAcquirePrcbLock(IN PKPRCB Prcb) { - /* Check if the wait is alertable */ - if (Alertable) - { - /* It is, first check if the thread is alerted in this mode */ - if (Thread->Alerted[WaitMode]) - { - /* It is, so bail out of the wait */ - Thread->Alerted[WaitMode] = FALSE; - return STATUS_ALERTED; - } - else if ((WaitMode != KernelMode) && - (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode]))) - { - /* It's isn't, but this is a user wait with queued user APCs */ - Thread->ApcState.UserApcPending = TRUE; - return STATUS_USER_APC; - } - else if (Thread->Alerted[KernelMode]) - { - /* It isn't that either, but we're alered in kernel mode */ - Thread->Alerted[KernelMode] = FALSE; - return STATUS_ALERTED; - } - } - else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending)) - { - /* Not alertable, but this is a user wait with pending user APCs */ - return STATUS_USER_APC; - } - - /* Otherwise, we're fine */ - return STATUS_WAIT_0; + UNREFERENCED_PARAMETER(Prcb); } -VOID +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// FORCEINLINE -KxSetTimerForThreadWait(IN PKTIMER Timer, - IN LARGE_INTEGER Interval) +VOID +KiReleasePrcbLock(IN PKPRCB Prcb) { - LARGE_INTEGER InterruptTime, SystemTime, TimeDifference; + UNREFERENCED_PARAMETER(Prcb); +} + +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// +FORCEINLINE +VOID +KiAcquireThreadLock(IN PKTHREAD Thread) +{ + UNREFERENCED_PARAMETER(Thread); +} + +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// +FORCEINLINE +VOID +KiReleaseThreadLock(IN PKTHREAD Thread) +{ + UNREFERENCED_PARAMETER(Thread); +} + +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// +FORCEINLINE +BOOLEAN +KiTryThreadLock(IN PKTHREAD Thread) +{ + UNREFERENCED_PARAMETER(Thread); + return FALSE; +} + +FORCEINLINE +VOID +KiCheckDeferredReadyList(IN PKPRCB Prcb) +{ + /* There are no deferred ready lists on UP systems */ + UNREFERENCED_PARAMETER(Prcb); +} + +FORCEINLINE +VOID +KiRundownThread(IN PKTHREAD Thread) +{ + /* Check if this is the NPX Thread */ + if (KeGetCurrentPrcb()->NpxThread == Thread) + { + /* Clear it */ + KeGetCurrentPrcb()->NpxThread = NULL; + Ke386FnInit(); + } +} + +FORCEINLINE +VOID +KiRequestApcInterrupt(IN BOOLEAN NeedApc, + IN UCHAR Processor) +{ + /* We deliver instantly on UP */ + UNREFERENCED_PARAMETER(NeedApc); + UNREFERENCED_PARAMETER(Processor); +} + +FORCEINLINE +PKSPIN_LOCK_QUEUE +KiAcquireTimerLock(IN ULONG Hand) +{ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Nothing to do on UP */ + UNREFERENCED_PARAMETER(Hand); + return NULL; +} + +FORCEINLINE +VOID +KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) +{ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Nothing to do on UP */ + UNREFERENCED_PARAMETER(LockQueue); +} + +#else + +// +// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +{ + for (;;) + { + /* Try to acquire it */ + if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) + { + /* Value changed... wait until it's locked */ + while (*(volatile KSPIN_LOCK *)SpinLock == 1) + { +#ifdef DBG + /* On debug builds, we use a much slower but useful routine */ + Kii386SpinOnSpinLock(SpinLock, 5); +#else + /* Otherwise, just yield and keep looping */ + YieldProcessor(); +#endif + } + } + else + { +#ifdef DBG + /* On debug builds, we OR in the KTHREAD */ + *SpinLock = KeGetCurrentThread() | 1; +#endif + /* All is well, break out */ + break; + } + } +} + +// +// Spinlock Release at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +{ +#ifdef DBG + /* Make sure that the threads match */ + if ((KeGetCurrentThread() | 1) != *SpinLock) + { + /* They don't, bugcheck */ + KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0); + } +#endif + /* Clear the lock */ + InterlockedAnd(SpinLock, 0); +} + +KIRQL +FORCEINLINE +KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + LONG OldValue, NewValue; + + /* Make sure we're at a safe level to touch the lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Start acquire loop */ + do + { + /* Loop until the other CPU releases it */ + while ((UCHAR)Object->Lock & KOBJECT_LOCK_BIT) + { + /* Let the CPU know that this is a loop */ + YieldProcessor(); + }; + + /* Try acquiring the lock now */ + NewValue = InterlockedCompareExchange(&Object->Lock, + OldValue | KOBJECT_LOCK_BIT, + OldValue); + } while (NewValue != OldValue); +} + +KIRQL +FORCEINLINE +KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + /* Make sure we're at a safe level to touch the lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Release it */ + InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT); +} + +KIRQL +FORCEINLINE +KiAcquireDispatcherLock(VOID) +{ + /* Raise to synchronization level and acquire the dispatcher lock */ + return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock); +} + +VOID +FORCEINLINE +KiReleaseDispatcherLock(IN KIRQL OldIrql) +{ + /* First release the lock */ + KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> + LockQueue[LockQueueDispatcherLock]); + + /* Then exit the dispatcher */ + KiExitDispatcher(OldIrql); +} + +// +// This routine inserts a thread into the deferred ready list of the given CPU +// +FORCEINLINE +VOID +KiInsertDeferredReadyList(IN PKTHREAD Thread) +{ + PKPRCB Prcb = KeGetCurrentPrcb(); + + /* Set the thread to deferred state and CPU */ + Thread->State = DeferredReady; + Thread->DeferredProcessor = Prcb->Number; + + /* Add it on the list */ + PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry); +} + +FORCEINLINE +VOID +KiRescheduleThread(IN BOOLEAN NewThread, + IN ULONG Cpu) +{ + /* Check if a new thread needs to be scheduled on a different CPU */ + if ((NewThread) && !(KeGetPcr()->Number == Cpu)) + { + /* Send an IPI to request delivery */ + KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC); + } +} + +// +// This routine sets the current thread in a swap busy state, which ensure that +// nobody else tries to swap it concurrently. +// +FORCEINLINE +VOID +KiSetThreadSwapBusy(IN PKTHREAD Thread) +{ + /* Make sure nobody already set it */ + ASSERT(Thread->SwapBusy == FALSE); + + /* Set it ourselves */ + Thread->SwapBusy = TRUE; +} + +// +// This routine acquires the PRCB lock so that only one caller can touch +// volatile PRCB data. +// +// Since this is a simple optimized spin-lock, it must be be only acquired +// at dispatcher level or higher! +// +FORCEINLINE +VOID +KiAcquirePrcbLock(IN PKPRCB Prcb) +{ + /* Make sure we're at a safe level to touch the PRCB lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Start acquire loop */ + for (;;) + { + /* Acquire the lock and break out if we acquired it first */ + if (!InterlockedExchange(&Prcb->PrcbLock, 1)) break; + + /* Loop until the other CPU releases it */ + do + { + /* Let the CPU know that this is a loop */ + YieldProcessor(); + } while (Prcb->PrcbLock); + } +} + +// +// This routine releases the PRCB lock so that other callers can touch +// volatile PRCB data. +// +// Since this is a simple optimized spin-lock, it must be be only acquired +// at dispatcher level or higher! +// +FORCEINLINE +VOID +KiReleasePrcbLock(IN PKPRCB Prcb) +{ + /* Make sure it's acquired! */ + ASSERT(Prcb->PrcbLock != 0); + + /* Release it */ + InterlockedAnd(&Prcb->PrcbLock, 0); +} + +// +// This routine acquires the thread lock so that only one caller can touch +// volatile thread data. +// +// Since this is a simple optimized spin-lock, it must be be only acquired +// at dispatcher level or higher! +// +FORCEINLINE +VOID +KiAcquireThreadLock(IN PKTHREAD Thread) +{ + /* Make sure we're at a safe level to touch the thread lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); - /* Check the timer's interval to see if it's absolute */ - Timer->Header.Absolute = FALSE; - if (!Timer->Period) Timer->Header.SignalState = FALSE; - if (Interval.HighPart >= 0) + /* Start acquire loop */ + for (;;) { - /* Get the system time and calculate the relative time */ - KeQuerySystemTime(&SystemTime); - TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart; - Timer->Header.Absolute = TRUE; + /* Acquire the lock and break out if we acquired it first */ + if (!InterlockedExchange(&Thread->ThreadLock, 1)) break; - /* Check if we've already expired */ - if (TimeDifference.HighPart >= 0) - { - /* Reset everything */ - Timer->DueTime.QuadPart = 0; - Timer->Header.SignalState = TRUE; - return; - } - else + /* Loop until the other CPU releases it */ + do { - /* Update the interval */ - Interval = TimeDifference; - } + /* Let the CPU know that this is a loop */ + YieldProcessor(); + } while (Thread->ThreadLock); } - - /* Calculate the due time */ - InterruptTime.QuadPart = KeQueryInterruptTime(); - Timer->DueTime.QuadPart = InterruptTime.QuadPart - Interval.QuadPart; } -#define KxDelayThreadWait() \ - \ - /* Setup the Wait Block */ \ - Thread->WaitBlockList = TimerBlock; \ - \ - /* Setup the timer */ \ - KxSetTimerForThreadWait(Timer, *Interval); \ - \ - /* Save the due time for the caller */ \ - DueTime.QuadPart = Timer->DueTime.QuadPart; \ - \ - /* Link the timer to this Wait Block */ \ - TimerBlock->NextWaitBlock = TimerBlock; \ - Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ - Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ - \ - /* Clear wait status */ \ - Thread->WaitStatus = STATUS_SUCCESS; \ - \ - /* Setup wait fields */ \ - Thread->Alertable = Alertable; \ - Thread->WaitReason = DelayExecution; \ - Thread->WaitMode = WaitMode; \ - \ - /* Check if we can swap the thread's stack */ \ - Thread->WaitListEntry.Flink = NULL; \ - Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ - \ - /* Set the wait time */ \ - Thread->WaitTime = KeTickCount.LowPart; - -#define KxMultiThreadWait() \ - /* Link wait block array to the thread */ \ - Thread->WaitBlockList = WaitBlockArray; \ - \ - /* Reset the index */ \ - Index = 0; \ - \ - /* Loop wait blocks */ \ - do \ - { \ - /* Fill out the wait block */ \ - WaitBlock = &WaitBlockArray[Index]; \ - WaitBlock->Object = Object[Index]; \ - WaitBlock->WaitKey = (USHORT)Index; \ - WaitBlock->WaitType = WaitType; \ - WaitBlock->Thread = Thread; \ - \ - /* Link to next block */ \ - WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \ - Index++; \ - } while (Index < Count); \ - \ - /* Link the last block */ \ - WaitBlock->NextWaitBlock = WaitBlockArray; \ - \ - /* Set default wait status */ \ - Thread->WaitStatus = STATUS_WAIT_0; \ - \ - /* Check if we have a timer */ \ - if (Timeout) \ - { \ - /* Link to the block */ \ - TimerBlock->NextWaitBlock = WaitBlockArray; \ - \ - /* Setup the timer */ \ - KxSetTimerForThreadWait(Timer, *Timeout); \ - \ - /* Save the due time for the caller */ \ - DueTime.QuadPart = Timer->DueTime.QuadPart; \ - \ - /* Initialize the list */ \ - InitializeListHead(&Timer->Header.WaitListHead); \ - } \ - \ - /* Set wait settings */ \ - Thread->Alertable = Alertable; \ - Thread->WaitMode = WaitMode; \ - Thread->WaitReason = WaitReason; \ - \ - /* Check if we can swap the thread's stack */ \ - Thread->WaitListEntry.Flink = NULL; \ - Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ - \ - /* Set the wait time */ \ - Thread->WaitTime = KeTickCount.LowPart; - -#define KxSingleThreadWait() \ - /* Setup the Wait Block */ \ - Thread->WaitBlockList = WaitBlock; \ - WaitBlock->WaitKey = STATUS_SUCCESS; \ - WaitBlock->Object = Object; \ - WaitBlock->WaitType = WaitAny; \ - \ - /* Clear wait status */ \ - Thread->WaitStatus = STATUS_SUCCESS; \ - \ - /* Check if we have a timer */ \ - if (Timeout) \ - { \ - /* Setup the timer */ \ - KxSetTimerForThreadWait(Timer, *Timeout); \ - \ - /* Save the due time for the caller */ \ - DueTime.QuadPart = Timer->DueTime.QuadPart; \ - \ - /* Pointer to timer block */ \ - WaitBlock->NextWaitBlock = TimerBlock; \ - TimerBlock->NextWaitBlock = WaitBlock; \ - \ - /* Link the timer to this Wait Block */ \ - Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ - Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ - } \ - else \ - { \ - /* No timer block, just ourselves */ \ - WaitBlock->NextWaitBlock = WaitBlock; \ - } \ - \ - /* Set wait settings */ \ - Thread->Alertable = Alertable; \ - Thread->WaitMode = WaitMode; \ - Thread->WaitReason = WaitReason; \ - \ - /* Check if we can swap the thread's stack */ \ - Thread->WaitListEntry.Flink = NULL; \ - Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ - \ - /* Set the wait time */ \ - Thread->WaitTime = KeTickCount.LowPart; - -#define KxQueueThreadWait() \ - /* Setup the Wait Block */ \ - Thread->WaitBlockList = WaitBlock; \ - WaitBlock->WaitKey = STATUS_SUCCESS; \ - WaitBlock->Object = Queue; \ - WaitBlock->WaitType = WaitAny; \ - WaitBlock->Thread = Thread; \ - \ - /* Clear wait status */ \ - Thread->WaitStatus = STATUS_SUCCESS; \ - \ - /* Check if we have a timer */ \ - if (Timeout) \ - { \ - /* Setup the timer */ \ - KxSetTimerForThreadWait(Timer, *Timeout); \ - \ - /* Save the due time for the caller */ \ - DueTime.QuadPart = Timer->DueTime.QuadPart; \ - \ - /* Pointer to timer block */ \ - WaitBlock->NextWaitBlock = TimerBlock; \ - TimerBlock->NextWaitBlock = WaitBlock; \ - \ - /* Link the timer to this Wait Block */ \ - Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ - Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ - } \ - else \ - { \ - /* No timer block, just ourselves */ \ - WaitBlock->NextWaitBlock = WaitBlock; \ - } \ - \ - /* Set wait settings */ \ - Thread->Alertable = FALSE; \ - Thread->WaitMode = WaitMode; \ - Thread->WaitReason = WrQueue; \ - \ - /* Check if we can swap the thread's stack */ \ - Thread->WaitListEntry.Flink = NULL; \ - Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ - \ - /* Set the wait time */ \ - Thread->WaitTime = KeTickCount.LowPart; - // -// Unwaits a Thread +// This routine releases the thread lock so that other callers can touch +// volatile thread data. +// +// Since this is a simple optimized spin-lock, it must be be only acquired +// at dispatcher level or higher! // FORCEINLINE VOID -KxUnwaitThread(IN DISPATCHER_HEADER *Object, - IN KPRIORITY Increment) +KiReleaseThreadLock(IN PKTHREAD Thread) { - PLIST_ENTRY WaitEntry, WaitList; - PKWAIT_BLOCK WaitBlock; - PKTHREAD WaitThread; - ULONG WaitKey; - - /* Loop the Wait Entries */ - WaitList = &Object->WaitListHead; - ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE); - WaitEntry = WaitList->Flink; - do - { - /* Get the current wait block */ - WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); + /* Release it */ + InterlockedAnd(&Thread->ThreadLock, 0); +} - /* Get the waiting thread */ - WaitThread = WaitBlock->Thread; +FORCEINLINE +BOOLEAN +KiTryThreadLock(IN PKTHREAD Thread) +{ + LONG Value; - /* Check the current Wait Mode */ - if (WaitBlock->WaitType == WaitAny) - { - /* Use the actual wait key */ - WaitKey = WaitBlock->WaitKey; - } - else - { - /* Otherwise, use STATUS_KERNEL_APC */ - WaitKey = STATUS_KERNEL_APC; - } + /* If the lock isn't acquired, return false */ + if (!Thread->ThreadLock) return FALSE; - /* Unwait the thread */ - KiUnwaitThread(WaitThread, WaitKey, Increment); + /* Otherwise, try to acquire it and check the result */ + Value = 1; + Value = InterlockedExchange(&Thread->ThreadLock, &Value); - /* Next entry */ - WaitEntry = WaitList->Flink; - } while (WaitEntry != WaitList); + /* Return the lock state */ + return (Value == TRUE); } -// -// Unwaits a Thread waiting on an event -// FORCEINLINE VOID -KxUnwaitThreadForEvent(IN PKEVENT Event, - IN KPRIORITY Increment) +KiCheckDeferredReadyList(IN PKPRCB Prcb) { - PLIST_ENTRY WaitEntry, WaitList; - PKWAIT_BLOCK WaitBlock; - PKTHREAD WaitThread; + /* Scan the deferred ready lists if required */ + if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb); +} - /* Loop the Wait Entries */ - WaitList = &Event->Header.WaitListHead; - ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); - WaitEntry = WaitList->Flink; - do +FORCEINLINE +VOID +KiRequestApcInterrupt(IN BOOLEAN NeedApc, + IN UCHAR Processor) +{ + /* Check if we need to request APC delivery */ + if (NeedApc) { - /* Get the current wait block */ - WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); - - /* Get the waiting thread */ - WaitThread = WaitBlock->Thread; - - /* Check the current Wait Mode */ - if (WaitBlock->WaitType == WaitAny) + /* Check if it's on another CPU */ + if (KeGetPcr()->Number != Cpu) { - /* Un-signal it */ - Event->Header.SignalState = 0; - - /* Un-signal the event and unwait the thread */ - KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment); - break; + /* Send an IPI to request delivery */ + KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC); } - - /* Unwait the thread with STATUS_KERNEL_APC */ - KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment); - - /* Next entry */ - WaitEntry = WaitList->Flink; - } while (WaitEntry != WaitList); + else + { + /* Request a software interrupt */ + HalRequestSoftwareInterrupt(APC_LEVEL); + } + } } -#ifndef _CONFIG_SMP -// -// Spinlock Acquire at IRQL >= DISPATCH_LEVEL -// +#endif + FORCEINLINE VOID -KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +KiAcquireApcLock(IN PKTHREAD Thread, + IN PKLOCK_QUEUE_HANDLE Handle) { - /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ - UNREFERENCED_PARAMETER(SpinLock); + /* Acquire the lock and raise to synchronization level */ + KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle); } -// -// Spinlock Release at IRQL >= DISPATCH_LEVEL -// FORCEINLINE VOID -KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread, + IN PKLOCK_QUEUE_HANDLE Handle) { - /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ - UNREFERENCED_PARAMETER(SpinLock); + /* Acquire the lock */ + KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle); } -// -// This routine protects against multiple CPU acquires, it's meaningless on UP. -// -VOID FORCEINLINE -KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) +VOID +KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread, + IN PKLOCK_QUEUE_HANDLE Handle) { - UNREFERENCED_PARAMETER(Object); + /* Acquire the lock */ + KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle); } -// -// This routine protects against multiple CPU acquires, it's meaningless on UP. -// -VOID FORCEINLINE -KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) +VOID +KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle) { - UNREFERENCED_PARAMETER(Object); + /* Release the lock */ + KeReleaseInStackQueuedSpinLock(Handle); } -KIRQL FORCEINLINE -KiAcquireDispatcherLock(VOID) +VOID +KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) { - /* Raise to DPC level */ - return KeRaiseIrqlToDpcLevel(); + /* Release the lock */ + KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); } -VOID FORCEINLINE -KiReleaseDispatcherLock(IN KIRQL OldIrql) +VOID +KiAcquireProcessLock(IN PKPROCESS Process, + IN PKLOCK_QUEUE_HANDLE Handle) { - /* Just exit the dispatcher */ - KiExitDispatcher(OldIrql); + /* Acquire the lock and raise to synchronization level */ + KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle); } -VOID FORCEINLINE -KiAcquireDispatcherLockAtDpcLevel(VOID) +VOID +KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle) { - /* This is a no-op at DPC Level for UP systems */ - return; + /* Release the lock */ + KeReleaseInStackQueuedSpinLock(Handle); } -VOID FORCEINLINE -KiReleaseDispatcherLockFromDpcLevel(VOID) +VOID +KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) { - /* This is a no-op at DPC Level for UP systems */ - return; + /* Release the lock */ + KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); } -// -// This routine makes the thread deferred ready on the boot CPU. -// FORCEINLINE VOID -KiInsertDeferredReadyList(IN PKTHREAD Thread) +KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, + IN PKLOCK_QUEUE_HANDLE DeviceLock) { - /* Set the thread to deferred state and boot CPU */ - Thread->State = DeferredReady; - Thread->DeferredProcessor = 0; - - /* Make the thread ready immediately */ - KiDeferredReadyThread(Thread); + /* Check if we were called from a threaded DPC */ + if (KeGetCurrentPrcb()->DpcThreadActive) + { + /* Lock the Queue, we're not at DPC level */ + KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock); + } + else + { + /* We must be at DPC level, acquire the lock safely */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock, + DeviceLock); + } } FORCEINLINE VOID -KiRescheduleThread(IN BOOLEAN NewThread, - IN ULONG Cpu) +KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock) { - /* This is meaningless on UP systems */ - UNREFERENCED_PARAMETER(NewThread); - UNREFERENCED_PARAMETER(Cpu); + /* Check if we were called from a threaded DPC */ + if (KeGetCurrentPrcb()->DpcThreadActive) + { + /* Unlock the Queue, we're not at DPC level */ + KeReleaseInStackQueuedSpinLock(DeviceLock); + } + else + { + /* We must be at DPC level, release the lock safely */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock); + } } // -// This routine protects against multiple CPU acquires, it's meaningless on UP. +// Satisfies the wait of any dispatcher object // -FORCEINLINE -VOID -KiSetThreadSwapBusy(IN PKTHREAD Thread) -{ - UNREFERENCED_PARAMETER(Thread); +#define KiSatisfyObjectWait(Object, Thread) \ +{ \ + /* Special case for Mutants */ \ + if ((Object)->Header.Type == MutantObject) \ + { \ + /* Decrease the Signal State */ \ + (Object)->Header.SignalState--; \ + \ + /* Check if it's now non-signaled */ \ + if (!(Object)->Header.SignalState) \ + { \ + /* Set the Owner Thread */ \ + (Object)->OwnerThread = Thread; \ + \ + /* Disable APCs if needed */ \ + Thread->KernelApcDisable = Thread->KernelApcDisable - \ + (Object)->ApcDisable; \ + \ + /* Check if it's abandoned */ \ + if ((Object)->Abandoned) \ + { \ + /* Unabandon it */ \ + (Object)->Abandoned = FALSE; \ + \ + /* Return Status */ \ + Thread->WaitStatus = STATUS_ABANDONED; \ + } \ + \ + /* Insert it into the Mutant List */ \ + InsertHeadList(Thread->MutantListHead.Blink, \ + &(Object)->MutantListEntry); \ + } \ + } \ + else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ + EventSynchronizationObject) \ + { \ + /* Synchronization Timers and Events just get un-signaled */ \ + (Object)->Header.SignalState = 0; \ + } \ + else if ((Object)->Header.Type == SemaphoreObject) \ + { \ + /* These ones can have multiple states, so we only decrease it */ \ + (Object)->Header.SignalState--; \ + } \ } // -// This routine protects against multiple CPU acquires, it's meaningless on UP. +// Satisfies the wait of a mutant dispatcher object // -FORCEINLINE -VOID -KiAcquirePrcbLock(IN PKPRCB Prcb) -{ - UNREFERENCED_PARAMETER(Prcb); +#define KiSatisfyMutantWait(Object, Thread) \ +{ \ + /* Decrease the Signal State */ \ + (Object)->Header.SignalState--; \ + \ + /* Check if it's now non-signaled */ \ + if (!(Object)->Header.SignalState) \ + { \ + /* Set the Owner Thread */ \ + (Object)->OwnerThread = Thread; \ + \ + /* Disable APCs if needed */ \ + Thread->KernelApcDisable = Thread->KernelApcDisable - \ + (Object)->ApcDisable; \ + \ + /* Check if it's abandoned */ \ + if ((Object)->Abandoned) \ + { \ + /* Unabandon it */ \ + (Object)->Abandoned = FALSE; \ + \ + /* Return Status */ \ + Thread->WaitStatus = STATUS_ABANDONED; \ + } \ + \ + /* Insert it into the Mutant List */ \ + InsertHeadList(Thread->MutantListHead.Blink, \ + &(Object)->MutantListEntry); \ + } \ } // -// This routine protects against multiple CPU acquires, it's meaningless on UP. +// Satisfies the wait of any nonmutant dispatcher object // -FORCEINLINE -VOID -KiReleasePrcbLock(IN PKPRCB Prcb) -{ - UNREFERENCED_PARAMETER(Prcb); +#define KiSatisfyNonMutantWait(Object) \ +{ \ + if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \ + EventSynchronizationObject) \ + { \ + /* Synchronization Timers and Events just get un-signaled */ \ + (Object)->Header.SignalState = 0; \ + } \ + else if ((Object)->Header.Type == SemaphoreObject) \ + { \ + /* These ones can have multiple states, so we only decrease it */ \ + (Object)->Header.SignalState--; \ + } \ } // -// This routine protects against multiple CPU acquires, it's meaningless on UP. +// Recalculates the due time // +PLARGE_INTEGER FORCEINLINE -VOID -KiAcquireThreadLock(IN PKTHREAD Thread) +KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, + IN PLARGE_INTEGER DueTime, + IN OUT PLARGE_INTEGER NewDueTime) { - UNREFERENCED_PARAMETER(Thread); -} + /* Don't do anything for absolute waits */ + if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime; -// -// This routine protects against multiple CPU acquires, it's meaningless on UP. -// -FORCEINLINE -VOID -KiReleaseThreadLock(IN PKTHREAD Thread) -{ - UNREFERENCED_PARAMETER(Thread); + /* Otherwise, query the interrupt time and recalculate */ + NewDueTime->QuadPart = KeQueryInterruptTime(); + NewDueTime->QuadPart -= DueTime->QuadPart; + return NewDueTime; } // -// This routine protects against multiple CPU acquires, it's meaningless on UP. +// Determines whether a thread should be added to the wait list // FORCEINLINE BOOLEAN -KiTryThreadLock(IN PKTHREAD Thread) -{ - UNREFERENCED_PARAMETER(Thread); - return FALSE; -} - -FORCEINLINE -VOID -KiCheckDeferredReadyList(IN PKPRCB Prcb) -{ - /* There are no deferred ready lists on UP systems */ - UNREFERENCED_PARAMETER(Prcb); -} - -FORCEINLINE -VOID -KiRundownThread(IN PKTHREAD Thread) +KiCheckThreadStackSwap(IN PKTHREAD Thread, + IN KPROCESSOR_MODE WaitMode) { - /* Check if this is the NPX Thread */ - if (KeGetCurrentPrcb()->NpxThread == Thread) + /* Check the required conditions */ + if ((WaitMode != KernelMode) && + (Thread->EnableStackSwap) && + (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9))) { - /* Clear it */ - KeGetCurrentPrcb()->NpxThread = NULL; - Ke386FnInit(); + /* We are go for swap */ + return TRUE; + } + else + { + /* Don't swap the thread */ + return FALSE; } } -FORCEINLINE -VOID -KiRequestApcInterrupt(IN BOOLEAN NeedApc, - IN UCHAR Processor) -{ - /* We deliver instantly on UP */ - UNREFERENCED_PARAMETER(NeedApc); - UNREFERENCED_PARAMETER(Processor); -} - -FORCEINLINE -PKSPIN_LOCK_QUEUE -KiAcquireTimerLock(IN ULONG Hand) -{ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); - - /* Nothing to do on UP */ - UNREFERENCED_PARAMETER(Hand); - return NULL; -} - -FORCEINLINE -VOID -KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue) -{ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); - - /* Nothing to do on UP */ - UNREFERENCED_PARAMETER(LockQueue); +// +// Adds a thread to the wait list +// +#define KiAddThreadToWaitList(Thread, Swappable) \ +{ \ + /* Make sure it's swappable */ \ + if (Swappable) \ + { \ + /* Insert it into the PRCB's List */ \ + InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \ + &Thread->WaitListEntry); \ + } \ } -#else - // -// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL +// Checks if a wait in progress should be interrupted by APCs or an alertable +// state. // FORCEINLINE -VOID -KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +NTSTATUS +KiCheckAlertability(IN PKTHREAD Thread, + IN BOOLEAN Alertable, + IN KPROCESSOR_MODE WaitMode) { - for (;;) + /* Check if the wait is alertable */ + if (Alertable) { - /* Try to acquire it */ - if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) + /* It is, first check if the thread is alerted in this mode */ + if (Thread->Alerted[WaitMode]) { - /* Value changed... wait until it's locked */ - while (*(volatile KSPIN_LOCK *)SpinLock == 1) - { -#ifdef DBG - /* On debug builds, we use a much slower but useful routine */ - Kii386SpinOnSpinLock(SpinLock, 5); -#else - /* Otherwise, just yield and keep looping */ - YieldProcessor(); -#endif - } + /* It is, so bail out of the wait */ + Thread->Alerted[WaitMode] = FALSE; + return STATUS_ALERTED; } - else + else if ((WaitMode != KernelMode) && + (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode]))) { -#ifdef DBG - /* On debug builds, we OR in the KTHREAD */ - *SpinLock = KeGetCurrentThread() | 1; -#endif - /* All is well, break out */ - break; + /* It's isn't, but this is a user wait with queued user APCs */ + Thread->ApcState.UserApcPending = TRUE; + return STATUS_USER_APC; + } + else if (Thread->Alerted[KernelMode]) + { + /* It isn't that either, but we're alered in kernel mode */ + Thread->Alerted[KernelMode] = FALSE; + return STATUS_ALERTED; } } -} - -// -// Spinlock Release at IRQL >= DISPATCH_LEVEL -// -FORCEINLINE -VOID -KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) -{ -#ifdef DBG - /* Make sure that the threads match */ - if ((KeGetCurrentThread() | 1) != *SpinLock) - { - /* They don't, bugcheck */ - KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0); - } -#endif - /* Clear the lock */ - InterlockedAnd(SpinLock, 0); -} - -KIRQL -FORCEINLINE -KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) -{ - LONG OldValue, NewValue; - - /* Make sure we're at a safe level to touch the lock */ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); - - /* Start acquire loop */ - do + else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending)) { - /* Loop until the other CPU releases it */ - while ((UCHAR)Object->Lock & KOBJECT_LOCK_BIT) - { - /* Let the CPU know that this is a loop */ - YieldProcessor(); - }; - - /* Try acquiring the lock now */ - NewValue = InterlockedCompareExchange(&Object->Lock, - OldValue | KOBJECT_LOCK_BIT, - OldValue); - } while (NewValue != OldValue); -} - -KIRQL -FORCEINLINE -KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) -{ - /* Make sure we're at a safe level to touch the lock */ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); - - /* Release it */ - InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT); -} - -KIRQL -FORCEINLINE -KiAcquireDispatcherLock(VOID) -{ - /* Raise to synchronization level and acquire the dispatcher lock */ - return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock); -} - -VOID -FORCEINLINE -KiReleaseDispatcherLock(IN KIRQL OldIrql) -{ - /* First release the lock */ - KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()-> - LockQueue[LockQueueDispatcherLock]); + /* Not alertable, but this is a user wait with pending user APCs */ + return STATUS_USER_APC; + } - /* Then exit the dispatcher */ - KiExitDispatcher(OldIrql); + /* Otherwise, we're fine */ + return STATUS_WAIT_0; } // -// This routine inserts a thread into the deferred ready list of the given CPU +// Called by Wait and Queue code to insert a timer for dispatching. +// Also called by KeSetTimerEx to insert a timer from the caller. // -FORCEINLINE VOID -KiInsertDeferredReadyList(IN PKTHREAD Thread) +FORCEINLINE +KxInsertTimer(IN PKTIMER Timer, + IN ULONG Hand) { - PKPRCB Prcb = KeGetCurrentPrcb(); - - /* Set the thread to deferred state and CPU */ - Thread->State = DeferredReady; - Thread->DeferredProcessor = Prcb->Number; + PKSPIN_LOCK_QUEUE LockQueue; - /* Add it on the list */ - PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry); -} + /* Acquire the lock and release the dispatcher lock */ + LockQueue = KiAcquireTimerLock(Hand); + KiReleaseDispatcherLockFromDpcLevel(); -FORCEINLINE -VOID -KiRescheduleThread(IN BOOLEAN NewThread, - IN ULONG Cpu) -{ - /* Check if a new thread needs to be scheduled on a different CPU */ - if ((NewThread) && !(KeGetPcr()->Number == Cpu)) + /* Try to insert the timer */ + if (KiInsertTimerTable(Timer, Hand)) { - /* Send an IPI to request delivery */ - KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC); + /* Complete it */ + KiCompleteTimer(Timer, LockQueue); + } + else + { + /* Do nothing, just release the lock */ + KiReleaseTimerLock(LockQueue); } } // -// This routine sets the current thread in a swap busy state, which ensure that -// nobody else tries to swap it concurrently. +// Called from Unlink and Queue Insert Code. +// Also called by timer code when canceling an inserted timer. +// Removes a timer from it's tree. // -FORCEINLINE VOID -KiSetThreadSwapBusy(IN PKTHREAD Thread) +FORCEINLINE +KxRemoveTreeTimer(IN PKTIMER Timer) { - /* Make sure nobody already set it */ - ASSERT(Thread->SwapBusy == FALSE); + ULONG Hand = Timer->Header.Hand; + PKSPIN_LOCK_QUEUE LockQueue; + PKTIMER_TABLE_ENTRY TimerEntry; - /* Set it ourselves */ - Thread->SwapBusy = TRUE; + /* Acquire timer lock */ + LockQueue = KiAcquireTimerLock(Hand); + + /* Set the timer as non-inserted */ + Timer->Header.Inserted = FALSE; + + /* Remove it from the timer list */ + if (RemoveEntryList(&Timer->TimerListEntry)) + { + /* Get the entry and check if it's empty */ + TimerEntry = &KiTimerTableListHead[Hand]; + if (IsListEmpty(&TimerEntry->Entry)) + { + /* Clear the time then */ + TimerEntry->Time.HighPart = 0xFFFFFFFF; + } + } + + /* Release the timer lock */ + KiReleaseTimerLock(LockQueue); } -// -// This routine acquires the PRCB lock so that only one caller can touch -// volatile PRCB data. -// -// Since this is a simple optimized spin-lock, it must be be only acquired -// at dispatcher level or higher! -// -FORCEINLINE VOID -KiAcquirePrcbLock(IN PKPRCB Prcb) +FORCEINLINE +KxSetTimerForThreadWait(IN PKTIMER Timer, + IN LARGE_INTEGER Interval, + OUT PULONG Hand) { - /* Make sure we're at a safe level to touch the PRCB lock */ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + ULONGLONG DueTime; + LARGE_INTEGER InterruptTime, SystemTime, TimeDifference; - /* Start acquire loop */ - for (;;) + /* Check the timer's interval to see if it's absolute */ + Timer->Header.Absolute = FALSE; + if (Interval.HighPart >= 0) { - /* Acquire the lock and break out if we acquired it first */ - if (!InterlockedExchange(&Prcb->PrcbLock, 1)) break; + /* Get the system time and calculate the relative time */ + KeQuerySystemTime(&SystemTime); + TimeDifference.QuadPart = SystemTime.QuadPart - Interval.QuadPart; + Timer->Header.Absolute = TRUE; - /* Loop until the other CPU releases it */ - do + /* Check if we've already expired */ + if (TimeDifference.HighPart >= 0) { - /* Let the CPU know that this is a loop */ - YieldProcessor(); - } while (Prcb->PrcbLock); + /* Reset everything */ + Timer->DueTime.QuadPart = 0; + *Hand = 0; + Timer->Header.Hand = 0; + return; + } + else + { + /* Update the interval */ + Interval = TimeDifference; + } } -} -// -// This routine releases the PRCB lock so that other callers can touch -// volatile PRCB data. -// -// Since this is a simple optimized spin-lock, it must be be only acquired -// at dispatcher level or higher! -// -FORCEINLINE -VOID -KiReleasePrcbLock(IN PKPRCB Prcb) -{ - /* Make sure it's acquired! */ - ASSERT(Prcb->PrcbLock != 0); + /* Calculate the due time */ + InterruptTime.QuadPart = KeQueryInterruptTime(); + DueTime = InterruptTime.QuadPart - Interval.QuadPart; + Timer->DueTime.QuadPart = DueTime; - /* Release it */ - InterlockedAnd(&Prcb->PrcbLock, 0); + /* Calculate the timer handle */ + *Hand = KiComputeTimerTableIndex(DueTime); + Timer->Header.Hand = (UCHAR)*Hand; } +#define KxDelayThreadWait() \ + \ + /* Setup the Wait Block */ \ + Thread->WaitBlockList = TimerBlock; \ + \ + /* Setup the timer */ \ + KxSetTimerForThreadWait(Timer, *Interval, &Hand); \ + \ + /* Save the due time for the caller */ \ + DueTime.QuadPart = Timer->DueTime.QuadPart; \ + \ + /* Link the timer to this Wait Block */ \ + TimerBlock->NextWaitBlock = TimerBlock; \ + Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ + Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ + \ + /* Clear wait status */ \ + Thread->WaitStatus = STATUS_SUCCESS; \ + \ + /* Setup wait fields */ \ + Thread->Alertable = Alertable; \ + Thread->WaitReason = DelayExecution; \ + Thread->WaitMode = WaitMode; \ + \ + /* Check if we can swap the thread's stack */ \ + Thread->WaitListEntry.Flink = NULL; \ + Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ + \ + /* Set the wait time */ \ + Thread->WaitTime = KeTickCount.LowPart; + +#define KxMultiThreadWait() \ + /* Link wait block array to the thread */ \ + Thread->WaitBlockList = WaitBlockArray; \ + \ + /* Reset the index */ \ + Index = 0; \ + \ + /* Loop wait blocks */ \ + do \ + { \ + /* Fill out the wait block */ \ + WaitBlock = &WaitBlockArray[Index]; \ + WaitBlock->Object = Object[Index]; \ + WaitBlock->WaitKey = (USHORT)Index; \ + WaitBlock->WaitType = WaitType; \ + WaitBlock->Thread = Thread; \ + \ + /* Link to next block */ \ + WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \ + Index++; \ + } while (Index < Count); \ + \ + /* Link the last block */ \ + WaitBlock->NextWaitBlock = WaitBlockArray; \ + \ + /* Set default wait status */ \ + Thread->WaitStatus = STATUS_WAIT_0; \ + \ + /* Check if we have a timer */ \ + if (Timeout) \ + { \ + /* Link to the block */ \ + TimerBlock->NextWaitBlock = WaitBlockArray; \ + \ + /* Setup the timer */ \ + KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ + \ + /* Save the due time for the caller */ \ + DueTime.QuadPart = Timer->DueTime.QuadPart; \ + \ + /* Initialize the list */ \ + InitializeListHead(&Timer->Header.WaitListHead); \ + } \ + \ + /* Set wait settings */ \ + Thread->Alertable = Alertable; \ + Thread->WaitMode = WaitMode; \ + Thread->WaitReason = WaitReason; \ + \ + /* Check if we can swap the thread's stack */ \ + Thread->WaitListEntry.Flink = NULL; \ + Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ + \ + /* Set the wait time */ \ + Thread->WaitTime = KeTickCount.LowPart; + +#define KxSingleThreadWait() \ + /* Setup the Wait Block */ \ + Thread->WaitBlockList = WaitBlock; \ + WaitBlock->WaitKey = STATUS_SUCCESS; \ + WaitBlock->Object = Object; \ + WaitBlock->WaitType = WaitAny; \ + \ + /* Clear wait status */ \ + Thread->WaitStatus = STATUS_SUCCESS; \ + \ + /* Check if we have a timer */ \ + if (Timeout) \ + { \ + /* Setup the timer */ \ + KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ + \ + /* Save the due time for the caller */ \ + DueTime.QuadPart = Timer->DueTime.QuadPart; \ + \ + /* Pointer to timer block */ \ + WaitBlock->NextWaitBlock = TimerBlock; \ + TimerBlock->NextWaitBlock = WaitBlock; \ + \ + /* Link the timer to this Wait Block */ \ + Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ + Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ + } \ + else \ + { \ + /* No timer block, just ourselves */ \ + WaitBlock->NextWaitBlock = WaitBlock; \ + } \ + \ + /* Set wait settings */ \ + Thread->Alertable = Alertable; \ + Thread->WaitMode = WaitMode; \ + Thread->WaitReason = WaitReason; \ + \ + /* Check if we can swap the thread's stack */ \ + Thread->WaitListEntry.Flink = NULL; \ + Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ + \ + /* Set the wait time */ \ + Thread->WaitTime = KeTickCount.LowPart; + +#define KxQueueThreadWait() \ + /* Setup the Wait Block */ \ + Thread->WaitBlockList = WaitBlock; \ + WaitBlock->WaitKey = STATUS_SUCCESS; \ + WaitBlock->Object = Queue; \ + WaitBlock->WaitType = WaitAny; \ + WaitBlock->Thread = Thread; \ + \ + /* Clear wait status */ \ + Thread->WaitStatus = STATUS_SUCCESS; \ + \ + /* Check if we have a timer */ \ + if (Timeout) \ + { \ + /* Setup the timer */ \ + KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \ + \ + /* Save the due time for the caller */ \ + DueTime.QuadPart = Timer->DueTime.QuadPart; \ + \ + /* Pointer to timer block */ \ + WaitBlock->NextWaitBlock = TimerBlock; \ + TimerBlock->NextWaitBlock = WaitBlock; \ + \ + /* Link the timer to this Wait Block */ \ + Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \ + Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \ + } \ + else \ + { \ + /* No timer block, just ourselves */ \ + WaitBlock->NextWaitBlock = WaitBlock; \ + } \ + \ + /* Set wait settings */ \ + Thread->Alertable = FALSE; \ + Thread->WaitMode = WaitMode; \ + Thread->WaitReason = WrQueue; \ + \ + /* Check if we can swap the thread's stack */ \ + Thread->WaitListEntry.Flink = NULL; \ + Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \ + \ + /* Set the wait time */ \ + Thread->WaitTime = KeTickCount.LowPart; + // -// This routine acquires the thread lock so that only one caller can touch -// volatile thread data. -// -// Since this is a simple optimized spin-lock, it must be be only acquired -// at dispatcher level or higher! +// Unwaits a Thread // FORCEINLINE VOID -KiAcquireThreadLock(IN PKTHREAD Thread) +KxUnwaitThread(IN DISPATCHER_HEADER *Object, + IN KPRIORITY Increment) { - /* Make sure we're at a safe level to touch the thread lock */ - ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + PLIST_ENTRY WaitEntry, WaitList; + PKWAIT_BLOCK WaitBlock; + PKTHREAD WaitThread; + ULONG WaitKey; - /* Start acquire loop */ - for (;;) + /* Loop the Wait Entries */ + WaitList = &Object->WaitListHead; + ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE); + WaitEntry = WaitList->Flink; + do { - /* Acquire the lock and break out if we acquired it first */ - if (!InterlockedExchange(&Thread->ThreadLock, 1)) break; - - /* Loop until the other CPU releases it */ - do - { - /* Let the CPU know that this is a loop */ - YieldProcessor(); - } while (Thread->ThreadLock); - } -} - -// -// This routine releases the thread lock so that other callers can touch -// volatile thread data. -// -// Since this is a simple optimized spin-lock, it must be be only acquired -// at dispatcher level or higher! -// -FORCEINLINE -VOID -KiReleaseThreadLock(IN PKTHREAD Thread) -{ - /* Release it */ - InterlockedAnd(&Thread->ThreadLock, 0); -} - -FORCEINLINE -BOOLEAN -KiTryThreadLock(IN PKTHREAD Thread) -{ - LONG Value; - - /* If the lock isn't acquired, return false */ - if (!Thread->ThreadLock) return FALSE; - - /* Otherwise, try to acquire it and check the result */ - Value = 1; - Value = InterlockedExchange(&Thread->ThreadLock, &Value); - - /* Return the lock state */ - return (Value == TRUE); -} + /* Get the current wait block */ + WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); -FORCEINLINE -VOID -KiCheckDeferredReadyList(IN PKPRCB Prcb) -{ - /* Scan the deferred ready lists if required */ - if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb); -} + /* Get the waiting thread */ + WaitThread = WaitBlock->Thread; -FORCEINLINE -VOID -KiRequestApcInterrupt(IN BOOLEAN NeedApc, - IN UCHAR Processor) -{ - /* Check if we need to request APC delivery */ - if (NeedApc) - { - /* Check if it's on another CPU */ - if (KeGetPcr()->Number != Cpu) + /* Check the current Wait Mode */ + if (WaitBlock->WaitType == WaitAny) { - /* Send an IPI to request delivery */ - KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC); + /* Use the actual wait key */ + WaitKey = WaitBlock->WaitKey; } else { - /* Request a software interrupt */ - HalRequestSoftwareInterrupt(APC_LEVEL); + /* Otherwise, use STATUS_KERNEL_APC */ + WaitKey = STATUS_KERNEL_APC; } - } -} - -#endif - -FORCEINLINE -VOID -KiAcquireApcLock(IN PKTHREAD Thread, - IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Acquire the lock and raise to synchronization level */ - KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle); -} -FORCEINLINE -VOID -KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread, - IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Acquire the lock */ - KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle); -} + /* Unwait the thread */ + KiUnwaitThread(WaitThread, WaitKey, Increment); -FORCEINLINE -VOID -KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread, - IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Acquire the lock */ - KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle); + /* Next entry */ + WaitEntry = WaitList->Flink; + } while (WaitEntry != WaitList); } +// +// Unwaits a Thread waiting on an event +// FORCEINLINE VOID -KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle) +KxUnwaitThreadForEvent(IN PKEVENT Event, + IN KPRIORITY Increment) { - /* Release the lock */ - KeReleaseInStackQueuedSpinLock(Handle); -} + PLIST_ENTRY WaitEntry, WaitList; + PKWAIT_BLOCK WaitBlock; + PKTHREAD WaitThread; -FORCEINLINE -VOID -KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Release the lock */ - KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); -} + /* Loop the Wait Entries */ + WaitList = &Event->Header.WaitListHead; + ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE); + WaitEntry = WaitList->Flink; + do + { + /* Get the current wait block */ + WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry); -FORCEINLINE -VOID -KiAcquireProcessLock(IN PKPROCESS Process, - IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Acquire the lock and raise to synchronization level */ - KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle); -} + /* Get the waiting thread */ + WaitThread = WaitBlock->Thread; -FORCEINLINE -VOID -KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Release the lock */ - KeReleaseInStackQueuedSpinLock(Handle); -} + /* Check the current Wait Mode */ + if (WaitBlock->WaitType == WaitAny) + { + /* Un-signal it */ + Event->Header.SignalState = 0; -FORCEINLINE -VOID -KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle) -{ - /* Release the lock */ - KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle); -} + /* Un-signal the event and unwait the thread */ + KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment); + break; + } -FORCEINLINE -VOID -KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue, - IN PKLOCK_QUEUE_HANDLE DeviceLock) -{ - /* Check if we were called from a threaded DPC */ - if (KeGetCurrentPrcb()->DpcThreadActive) - { - /* Lock the Queue, we're not at DPC level */ - KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock); - } - else - { - /* We must be at DPC level, acquire the lock safely */ - ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock, - DeviceLock); - } -} + /* Unwait the thread with STATUS_KERNEL_APC */ + KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment); -FORCEINLINE -VOID -KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock) -{ - /* Check if we were called from a threaded DPC */ - if (KeGetCurrentPrcb()->DpcThreadActive) - { - /* Unlock the Queue, we're not at DPC level */ - KeReleaseInStackQueuedSpinLock(DeviceLock); - } - else - { - /* We must be at DPC level, release the lock safely */ - ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock); - } + /* Next entry */ + WaitEntry = WaitList->Flink; + } while (WaitEntry != WaitList); } // @@ -1468,14 +1538,3 @@ KeGetPreviousMode(VOID) return KeGetCurrentThread()->PreviousMode; } -VOID -FORCEINLINE -KiInsertWaitTimer(IN PKTIMER Timer) -{ - /* Now insert it into the Timer List */ - InsertAscendingList(&KiTimerListHead, - Timer, - KTIMER, - TimerListEntry, - DueTime.QuadPart); -} diff --git a/reactos/ntoskrnl/ke/clock.c b/reactos/ntoskrnl/ke/clock.c index ee739922e6e..5df7bcc1f8e 100644 --- a/reactos/ntoskrnl/ke/clock.c +++ b/reactos/ntoskrnl/ke/clock.c @@ -27,7 +27,7 @@ /* GLOBALS ****************************************************************/ LARGE_INTEGER KeBootTime, KeBootTimeBias; -KDPC KiExpireTimerDpc; +KDPC KiTimerExpireDpc; BOOLEAN KiClockSetupComplete = FALSE; ULONG KiTimeLimitIsrMicroseconds; @@ -37,7 +37,6 @@ ULONG KiTimeLimitIsrMicroseconds; volatile KSYSTEM_TIME KeTickCount = {0}; volatile ULONG KiRawTicks = 0; LONG KiTickOffset = 0; -extern LIST_ENTRY KiTimerListHead; /* * The increment in the system clock every timer tick (in system time units) diff --git a/reactos/ntoskrnl/ke/dpc.c b/reactos/ntoskrnl/ke/dpc.c index e18eb0e418d..6e481eef1ca 100644 --- a/reactos/ntoskrnl/ke/dpc.c +++ b/reactos/ntoskrnl/ke/dpc.c @@ -2,7 +2,7 @@ * PROJECT: ReactOS Kernel * LICENSE: GPL - See COPYING in the top level directory * FILE: ntoskrnl/ke/dpc.c - * PURPOSE: Routines for CPU-level support + * PURPOSE: Deferred Procedure Call (DPC) Support * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) * Philip Susi (phreak@iag.net) * Eric Kohl (ekohl@abo.rhein-zeitung.de) @@ -25,14 +25,231 @@ FAST_MUTEX KiGenericCallDpcMutex; /* PRIVATE FUNCTIONS *********************************************************/ -// -// This routine executes at the end of a thread's quantum. -// If the thread's quantum has expired, then a new thread is attempted -// to be scheduled. -// -// If no candidate thread has been found, the routine will return, otherwise -// it will swap contexts to the next scheduled thread. -// +VOID +NTAPI +KiTimerExpiration(IN PKDPC Dpc, + IN PVOID DeferredContext, + IN PVOID SystemArgument1, + IN PVOID SystemArgument2) +{ + LARGE_INTEGER SystemTime, InterruptTime, Interval; + LONG Limit, Index, i; + ULONG Timers, ActiveTimers, DpcCalls; + PLIST_ENTRY ListHead, NextEntry; + PKTIMER_TABLE_ENTRY TimerEntry; + KIRQL OldIrql; + PKTIMER Timer; + PKDPC TimerDpc; + ULONG Period; + DPC_QUEUE_ENTRY DpcEntry[MAX_TIMER_DPCS]; + PKSPIN_LOCK_QUEUE LockQueue; + + /* Disable interrupts */ + _disable(); + + /* Query system and interrupt time */ + KeQuerySystemTime(&SystemTime); + InterruptTime.QuadPart = KeQueryInterruptTime(); + Limit = KeTickCount.LowPart; + + /* Bring interrupts back */ + _enable(); + + /* Get the index of the timer and normalize it */ + Index = PtrToLong(SystemArgument1); + if ((Limit - Index) >= TIMER_TABLE_SIZE) + { + /* Normalize it */ + Limit = Index + TIMER_TABLE_SIZE - 1; + } + + /* Setup index and actual limit */ + Index--; + Limit &= (TIMER_TABLE_SIZE - 1); + + /* Setup accounting data */ + DpcCalls = 0; + Timers = 24; + ActiveTimers = 4; + + /* Lock the Database and Raise IRQL */ + OldIrql = KiAcquireDispatcherLock(); + + /* Start expiration loop */ + do + { + /* Get the current index */ + Index = (Index + 1) & (TIMER_TABLE_SIZE - 1); + + /* Get list pointers and loop the list */ + ListHead = &KiTimerTableListHead[Index].Entry; + while (ListHead != ListHead->Flink) + { + /* Lock the timer and go to the next entry */ + LockQueue = KiAcquireTimerLock(Index); + NextEntry = ListHead->Flink; + + /* Get the current timer and check its due time */ + Timers--; + Timer = CONTAINING_RECORD(NextEntry, KTIMER, TimerListEntry); + if ((NextEntry != ListHead) && + (Timer->DueTime.QuadPart <= InterruptTime.QuadPart)) + { + /* It's expired, remove it */ + ActiveTimers--; + if (RemoveEntryList(&Timer->TimerListEntry)) + { + /* Get the entry and check if it's empty */ + TimerEntry = &KiTimerTableListHead[Timer->Header.Hand]; + if (IsListEmpty(&TimerEntry->Entry)) + { + /* Clear the time then */ + TimerEntry->Time.HighPart = 0xFFFFFFFF; + } + } + + /* Make it non-inserted, unlock it, and signal it */ + Timer->Header.Inserted = FALSE; + KiReleaseTimerLock(LockQueue); + Timer->Header.SignalState = 1; + + /* Get the DPC and period */ + TimerDpc = Timer->Dpc; + Period = Timer->Period; + + /* Check if there's any waiters */ + if (!IsListEmpty(&Timer->Header.WaitListHead)) + { + /* Check the type of event */ + if (Timer->Header.Type == TimerNotificationObject) + { + /* Unwait the thread */ + KxUnwaitThread(&Timer->Header, IO_NO_INCREMENT); + } + else + { + /* Otherwise unwait the thread and signal the timer */ + KxUnwaitThreadForEvent((PKEVENT)Timer, IO_NO_INCREMENT); + } + } + + /* Check if we have a period */ + if (Period) + { + /* Calculate the interval and insert the timer */ + Interval.QuadPart = Int32x32To64(Period, -10000); + while (!KiInsertTreeTimer(Timer, Interval)); + } + + /* Check if we have a DPC */ + if (TimerDpc) + { + /* Setup the DPC Entry */ + DpcEntry[DpcCalls].Dpc = TimerDpc; + DpcEntry[DpcCalls].Routine = TimerDpc->DeferredRoutine; + DpcEntry[DpcCalls].Context = TimerDpc->DeferredContext; + DpcCalls++; + } + + /* Check if we're done processing */ + if (!(ActiveTimers) || !(Timers)) + { + /* Release the dispatcher while doing DPCs */ + KiReleaseDispatcherLock(DISPATCH_LEVEL); + + /* Start looping all DPC Entries */ + for (i = 0; DpcCalls; DpcCalls--, i++) + { + /* Call the DPC */ + DpcEntry[i].Routine(DpcEntry[i].Dpc, + DpcEntry[i].Context, + UlongToPtr(SystemTime.LowPart), + UlongToPtr(SystemTime.HighPart)); + } + + /* Reset accounting */ + Timers = 24; + ActiveTimers = 4; + + /* Lock the dispatcher database */ + KiAcquireDispatcherLock(); + } + } + else + { + /* Check if the timer list is empty */ + if (NextEntry != ListHead) + { + /* Sanity check */ + ASSERT(KiTimerTableListHead[Index].Time.QuadPart <= + Timer->DueTime.QuadPart); + + /* Update the time */ + _disable(); + KiTimerTableListHead[Index].Time.QuadPart = + Timer->DueTime.QuadPart; + _enable(); + } + + /* Release the lock */ + KiReleaseTimerLock(LockQueue); + + /* Check if we've scanned all the timers we could */ + if (!Timers) + { + /* Release the dispatcher while doing DPCs */ + KiReleaseDispatcherLock(DISPATCH_LEVEL); + + /* Start looping all DPC Entries */ + for (i = 0; DpcCalls; DpcCalls--, i++) + { + /* Call the DPC */ + DpcEntry[i].Routine(DpcEntry[i].Dpc, + DpcEntry[i].Context, + UlongToPtr(SystemTime.LowPart), + UlongToPtr(SystemTime.HighPart)); + } + + /* Reset accounting */ + Timers = 24; + ActiveTimers = 4; + + /* Lock the dispatcher database */ + KiAcquireDispatcherLock(); + } + + /* Done looping */ + break; + } + } + } while (Index != Limit); + + /* Check if we still have DPC entries */ + if (DpcCalls) + { + /* Release the dispatcher while doing DPCs */ + KiReleaseDispatcherLock(DISPATCH_LEVEL); + + /* Start looping all DPC Entries */ + for (i = 0; DpcCalls; DpcCalls--, i++) + { + /* Call the DPC */ + DpcEntry[i].Routine(DpcEntry[i].Dpc, + DpcEntry[i].Context, + UlongToPtr(SystemTime.LowPart), + UlongToPtr(SystemTime.HighPart)); + } + + /* Lower IRQL if we need to */ + if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql); + } + else + { + /* Unlock the dispatcher */ + KiReleaseDispatcherLock(OldIrql); + } +} + VOID NTAPI KiQuantumEnd(VOID) @@ -146,6 +363,7 @@ KiRetireDpcList(IN PKPRCB Prcb) PKDPC Dpc; PKDEFERRED_ROUTINE DeferredRoutine; PVOID DeferredContext, SystemArgument1, SystemArgument2; + ULONG_PTR TimerHand; /* Main outer loop */ do @@ -156,8 +374,11 @@ KiRetireDpcList(IN PKPRCB Prcb) /* Check if this is a timer expiration request */ if (Prcb->TimerRequest) { - /* FIXME: Not yet implemented */ - ASSERT(FALSE); + TimerHand = Prcb->TimerHand; + Prcb->TimerRequest = 0; + _enable(); + KiTimerExpiration(NULL, NULL, (PVOID) TimerHand, NULL); + _disable(); } /* Loop while we have entries in the queue */ diff --git a/reactos/ntoskrnl/ke/i386/clock.S b/reactos/ntoskrnl/ke/i386/clock.S index 298d23c61d1..a2c54e93117 100644 --- a/reactos/ntoskrnl/ke/i386/clock.S +++ b/reactos/ntoskrnl/ke/i386/clock.S @@ -13,6 +13,45 @@ /* FUNCTIONS ******************************************************************/ +.globl _KiComputeTimerTableIndex@8 +.func KiComputeTimerTableIndex@8 +_KiComputeTimerTableIndex@8: + + /* Save registers */ + push ebx + + /* Make the first multiplication */ + mov eax, [esp+8] + mul dword ptr [_KiTimeIncrementReciprocal+4] + mov ebx, eax + mov ecx, edx + + /* Make the second multiplication */ + mov eax, [esp+12] + mul dword ptr [_KiTimeIncrementReciprocal] + add ebx, eax + adc ecx, edx + + /* Multiply by the reciprocal */ + mov eax, [esp+8] + mul dword ptr [_KiTimeIncrementReciprocal] + mov eax, [esp+12] + push edx + mul dword ptr [_KiTimeIncrementReciprocal+4] + pop edx + add edx, ebx + adc eax, ecx + + /* Shift the result and generate the index */ + mov cl, [_KiTimeIncrementShiftCount] + shr eax, cl + and eax, TIMER_TABLE_SIZE - 1 + + /* Return */ + pop ebx + ret 8 +.endfunc + .globl _KeUpdateRunTime@4 .func KeUpdateRunTime@4 _KeUpdateRunTime@4: @@ -214,13 +253,50 @@ _KeUpdateSystemTime@0: /* FIXME: HACK */ mov ds:[USER_SHARED_DATA], ecx + /* Get hand index and entry into the table */ + and eax, TIMER_TABLE_SIZE - 1 + shl eax, 4 + + /* Compare the due time */ + cmp esi, [eax+_KiTimerTableListHead+KTIMER_TABLE_TIME+4] + jb NextHand + ja TimerExpired + cmp edi, [eax+_KiTimerTableListHead+KTIMER_TABLE_TIME] + jnb TimerExpired + +NextHand: + /* Move to the next hand */ + inc ebx + mov eax, ebx + IncompleteTick: - /* FIXME: NASTY Queue DPC to handle registered timers */ - push 0 - push [esp+KTRAP_FRAME_EIP] - push offset _KiExpireTimerDpc - call _KeInsertQueueDpc@12 + /* Get hand index and entry into the table */ + and eax, TIMER_TABLE_SIZE - 1 + shl eax, 4 + + /* Compare the due time */ + cmp esi, [eax+_KiTimerTableListHead+KTIMER_TABLE_TIME+4] + jb DebugCheck + ja TimerExpired + cmp edi, [eax+_KiTimerTableListHead+KTIMER_TABLE_TIME] + jb DebugCheck + +TimerExpired: + + /* Check if expiration is active */ + mov ecx, [fs:KPCR_PRCB] + cmp dword ptr [ecx+KPRCB_TIMER_REQUEST], 0 + jne DebugCheck + + /* It's not, register it */ + mov [ecx+KPRCB_TIMER_REQUEST], esp + mov [ecx+KPRCB_TIMER_HAND], ebx + mov ecx, DISPATCH_LEVEL + call @HalRequestSoftwareInterrupt@4 + +DebugCheck: + /* FIXME: Check for KdDebuggerEnabled */ /* Check if this was a full tick */ cmp dword ptr _KiTickOffset, 0 diff --git a/reactos/ntoskrnl/ke/krnlinit.c b/reactos/ntoskrnl/ke/krnlinit.c index 0b709bc7682..4a3524069f3 100644 --- a/reactos/ntoskrnl/ke/krnlinit.c +++ b/reactos/ntoskrnl/ke/krnlinit.c @@ -67,8 +67,8 @@ KiInitSystem(VOID) KeInitializeSpinLock(&BugCheckCallbackLock); /* Initialize the Timer Expiration DPC */ - KeInitializeDpc(&KiExpireTimerDpc, KiExpireTimers, NULL); - KeSetTargetProcessorDpc(&KiExpireTimerDpc, 0); + KeInitializeDpc(&KiTimerExpireDpc, KiTimerExpiration, NULL); + KeSetTargetProcessorDpc(&KiTimerExpireDpc, 0); /* Initialize Profiling data */ KeInitializeSpinLock(&KiProfileLock); @@ -84,9 +84,6 @@ KiInitSystem(VOID) KiTimerTableListHead[i].Time.LowPart = 0; } - /* Initialize old-style list */ - InitializeListHead(&KiTimerListHead); - /* Initialize the Swap event and all swap lists */ KeInitializeEvent(&KiSwapEvent, SynchronizationEvent, FALSE); InitializeListHead(&KiProcessInSwapListHead); @@ -293,3 +290,5 @@ KeInitSystem(VOID) return TRUE; } + + diff --git a/reactos/ntoskrnl/ke/queue.c b/reactos/ntoskrnl/ke/queue.c index c41141caf20..a2bddee7070 100644 --- a/reactos/ntoskrnl/ke/queue.c +++ b/reactos/ntoskrnl/ke/queue.c @@ -75,6 +75,7 @@ KiInsertQueue(IN PKQUEUE Queue, PKTHREAD Thread = KeGetCurrentThread(); PKWAIT_BLOCK WaitBlock; PLIST_ENTRY WaitEntry; + PKTIMER Timer; ASSERT_QUEUE(Queue); /* Save the old state */ @@ -109,12 +110,8 @@ KiInsertQueue(IN PKQUEUE Queue, Thread->WaitReason = 0; /* Check if there's a Thread Timer */ - if (Thread->Timer.Header.Inserted) - { - /* Cancel the Thread Timer with the no-lock fastpath */ - Thread->Timer.Header.Inserted = FALSE; - RemoveEntryList(&Thread->Timer.TimerListEntry); - } + Timer = &Thread->Timer; + if (Timer->Header.Inserted) KxRemoveTreeTimer(Timer); /* Reschedule the Thread */ KiReadyThread(Thread); @@ -251,6 +248,7 @@ KeRemoveQueue(IN PKQUEUE Queue, BOOLEAN Swappable; PLARGE_INTEGER OriginalDueTime = Timeout; LARGE_INTEGER DueTime, NewDueTime, InterruptTime; + ULONG Hand = 0; ASSERT_QUEUE(Queue); ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL); @@ -390,7 +388,7 @@ KeRemoveQueue(IN PKQUEUE Queue, if (Timeout) { /* Insert it */ - KiInsertWaitTimer(Timer); + KxInsertTimer(Timer, Hand); } else { diff --git a/reactos/ntoskrnl/ke/timer.c b/reactos/ntoskrnl/ke/timer.c index 160f869cc05..cd32d9d2875 100644 --- a/reactos/ntoskrnl/ke/timer.c +++ b/reactos/ntoskrnl/ke/timer.c @@ -4,188 +4,242 @@ * FILE: ntoskrnl/ke/timer.c * PURPOSE: Handle Kernel Timers (Kernel-part of Executive Timers) * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) - * David Welch (welch@mcmail.com) */ -/* INCLUDES ***************************************************************/ +/* INCLUDES ******************************************************************/ #include #define NDEBUG -#include +#include -/* GLOBALS ****************************************************************/ +/* GLOBALS *******************************************************************/ +KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]; LARGE_INTEGER KiTimeIncrementReciprocal; UCHAR KiTimeIncrementShiftCount; -LIST_ENTRY KiTimerListHead; -KTIMER_TABLE_ENTRY KiTimerTableListHead[TIMER_TABLE_SIZE]; -#define SYSTEM_TIME_UNITS_PER_MSEC (10000) -/* PRIVATE FUNCTIONS ******************************************************/ - -VOID -NTAPI -KiRemoveTimer(IN PKTIMER Timer) -{ - /* Remove the timer */ - Timer->Header.Inserted = FALSE; - RemoveEntryList(&Timer->TimerListEntry); -} +/* PRIVATE FUNCTIONS *********************************************************/ -/* - * Note: This function is called with the Dispatcher Lock held. - */ BOOLEAN -NTAPI -KiInsertTimer(IN PKTIMER Timer, - IN LARGE_INTEGER DueTime) +FASTCALL +KiInsertTreeTimer(IN PKTIMER Timer, + IN LARGE_INTEGER Interval) { - LARGE_INTEGER SystemTime; - LARGE_INTEGER DifferenceTime; - ULONGLONG InterruptTime; - - /* Set default data */ - Timer->Header.Inserted = TRUE; - Timer->Header.Absolute = FALSE; - if (!Timer->Period) Timer->Header.SignalState = FALSE; + BOOLEAN Inserted = FALSE; + ULONG Hand = 0; + PKSPIN_LOCK_QUEUE LockQueue; + LONGLONG DueTime; + LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime; + PKTIMER_TABLE_ENTRY TimerEntry; /* Convert to relative time if needed */ - if (DueTime.HighPart >= 0) + Timer->Header.Absolute = FALSE; + if (Interval.HighPart >= 0) { /* Get System Time */ KeQuerySystemTime(&SystemTime); /* Do the conversion */ - DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart; + DifferenceTime.QuadPart = SystemTime.QuadPart - Interval.QuadPart; /* Make sure it hasn't already expired */ + Timer->Header.Absolute = TRUE; if (DifferenceTime.HighPart >= 0) { /* Cancel everything */ Timer->Header.SignalState = TRUE; - Timer->Header.Inserted = FALSE; + Timer->Header.Hand = 0; + Timer->DueTime.QuadPart = 0; return FALSE; } /* Set the time as Absolute */ - Timer->Header.Absolute = TRUE; - DueTime = DifferenceTime; + Interval = DifferenceTime; } /* Get the Interrupt Time */ - InterruptTime = KeQueryInterruptTime(); - - /* Set the Final Due Time */ - Timer->DueTime.QuadPart = InterruptTime - DueTime.QuadPart; - - /* Now insert it into the Timer List */ - InsertAscendingList(&KiTimerListHead, - Timer, - KTIMER, - TimerListEntry, - DueTime.QuadPart); - return TRUE; + InterruptTime.QuadPart = KeQueryInterruptTime(); + + /* Recalculate due time */ + DueTime = InterruptTime.QuadPart - Interval.QuadPart; + Timer->DueTime.QuadPart = DueTime; + + /* Get the handle */ + Hand = KiComputeTimerTableIndex(DueTime); + Timer->Header.Hand = (UCHAR)Hand; + Timer->Header.Inserted = TRUE; + + /* Acquire the lock */ + LockQueue = KiAcquireTimerLock(Hand); + + /* Insert the timer */ + if (KiInsertTimerTable(Timer, Hand)) + { + /* It was already there, remove it */ + if (RemoveEntryList(&Timer->TimerListEntry)) + { + /* Get the entry and check if it's empty */ + TimerEntry = &KiTimerTableListHead[Hand]; + if (IsListEmpty(&TimerEntry->Entry)) + { + /* Clear the time then */ + TimerEntry->Time.HighPart = 0xFFFFFFFF; + } + } + } + else + { + /* Otherwise, we're now inserted */ + Inserted = TRUE; + } + + /* Release the lock and return insert status */ + return Inserted; } -/* - * We enter this function at IRQL DISPATCH_LEVEL, and with the - * Dispatcher Lock held! - */ -VOID -NTAPI -KiHandleExpiredTimer(IN PKTIMER Timer) +BOOLEAN +FASTCALL +KiInsertTimerTable(IN PKTIMER Timer, + IN ULONG Hand) +{ + LARGE_INTEGER InterruptTime; + LONGLONG DueTime = Timer->DueTime.QuadPart; + BOOLEAN Expired = FALSE; + PLIST_ENTRY ListHead, NextEntry; + PKTIMER CurrentTimer; + + /* Check if the period is zero */ + if (!Timer->Period) Timer->Header.SignalState = FALSE; + + /* Sanity check */ + ASSERT(Hand == KiComputeTimerTableIndex(DueTime)); + + /* Loop the timer list backwards */ + ListHead = &KiTimerTableListHead[Hand].Entry; + NextEntry = ListHead->Blink; + while (NextEntry != ListHead) + { + /* Get the timer */ + CurrentTimer = CONTAINING_RECORD(NextEntry, KTIMER, TimerListEntry); + + /* Now check if we can fit it before */ + if ((ULONGLONG)DueTime >= CurrentTimer->DueTime.QuadPart) break; + + /* Keep looping */ + NextEntry = NextEntry->Blink; + } + + /* Looped all the list, insert it here and get the interrupt time again */ + InsertHeadList(NextEntry, &Timer->TimerListEntry); + + /* Check if we didn't find it in the list */ + if (NextEntry == ListHead) + { + /* Set the time */ + KiTimerTableListHead[Hand].Time.QuadPart = DueTime; + + /* Make sure it hasn't expired already */ + InterruptTime.QuadPart = KeQueryInterruptTime(); + if (DueTime <= InterruptTime.QuadPart) Expired = TRUE; + } + + /* Return expired state */ + return Expired; +} + +BOOLEAN +FASTCALL +KiSignalTimer(IN PKTIMER Timer) { - LARGE_INTEGER DueTime; + BOOLEAN RequestInterrupt = FALSE; + PKDPC Dpc = Timer->Dpc; + ULONG Period = Timer->Period; + LARGE_INTEGER Interval, SystemTime; - /* Set it as Signaled */ + /* Set default values */ + Timer->Header.Inserted = FALSE; Timer->Header.SignalState = TRUE; - /* Check if it has any waiters */ + /* Check if the timer has waiters */ if (!IsListEmpty(&Timer->Header.WaitListHead)) { - /* Wake them */ - KiWaitTest(Timer, IO_NO_INCREMENT); + /* Check the type of event */ + if (Timer->Header.Type == TimerNotificationObject) + { + /* Unwait the thread */ + KxUnwaitThread(&Timer->Header, IO_NO_INCREMENT); + } + else + { + /* Otherwise unwait the thread and signal the timer */ + KxUnwaitThreadForEvent((PKEVENT)Timer, IO_NO_INCREMENT); + } } - /* If the Timer is periodic, reinsert the timer with the new due time */ - if (Timer->Period) + /* Check if we have a period */ + if (Period) { - /* Reinsert the Timer */ - DueTime.QuadPart = Timer->Period * -SYSTEM_TIME_UNITS_PER_MSEC; - while (!KiInsertTimer(Timer, DueTime)); + /* Calculate the interval and insert the timer */ + Interval.QuadPart = Int32x32To64(Period, -10000); + while (!KiInsertTreeTimer(Timer, Interval)); } - /* Check if the Timer has a DPC */ - if (Timer->Dpc) + /* Check if we have a DPC */ + if (Dpc) { - /* Insert the DPC */ - KeInsertQueueDpc(Timer->Dpc, - NULL, - NULL); + /* Insert it in the queue */ + KeQuerySystemTime(&SystemTime); + KeInsertQueueDpc(Dpc, + ULongToPtr(SystemTime.LowPart), + ULongToPtr(SystemTime.HighPart)); + RequestInterrupt = TRUE; } + + /* Return whether we need to request a DPC interrupt or not */ + return RequestInterrupt; } VOID -NTAPI -KiExpireTimers(IN PKDPC Dpc, - IN PVOID DeferredContext, - IN PVOID SystemArgument1, - IN PVOID SystemArgument2) +FASTCALL +KiCompleteTimer(IN PKTIMER Timer, + IN PKSPIN_LOCK_QUEUE LockQueue) { - PKTIMER Timer; - ULONGLONG InterruptTime; - LIST_ENTRY ExpiredTimerList; - PLIST_ENTRY ListHead, NextEntry; - KIRQL OldIrql; - - /* Initialize the Expired Timer List */ - InitializeListHead(&ExpiredTimerList); + LIST_ENTRY ListHead; + PKTIMER_TABLE_ENTRY TimerEntry; + BOOLEAN RequestInterrupt = FALSE; - /* Lock the Database and Raise IRQL */ - OldIrql = KiAcquireDispatcherLock(); + /* Remove it from the timer list */ + if (RemoveEntryList(&Timer->TimerListEntry)) + { + /* Get the entry and check if it's empty */ + TimerEntry = &KiTimerTableListHead[Timer->Header.Hand]; + if (IsListEmpty(&TimerEntry->Entry)) + { + /* Clear the time then */ + TimerEntry->Time.HighPart = 0xFFFFFFFF; + } + } - /* Query Interrupt Times */ - InterruptTime = KeQueryInterruptTime(); + /* Link the timer list to our stack */ + ListHead.Flink = &Timer->TimerListEntry; + ListHead.Blink = &Timer->TimerListEntry; + Timer->TimerListEntry.Flink = &ListHead; + Timer->TimerListEntry.Blink = &ListHead; - /* Loop through the Timer List */ - ListHead = &KiTimerListHead; - NextEntry = ListHead->Flink; - while (NextEntry != ListHead) - { - /* Get the timer */ - Timer = CONTAINING_RECORD(NextEntry, KTIMER, TimerListEntry); + /* Release the timer lock */ + KiReleaseTimerLock(LockQueue); - /* Check if we have to Expire it */ - if (InterruptTime < Timer->DueTime.QuadPart) break; + /* Acquire dispatcher lock */ + KiAcquireDispatcherLockAtDpcLevel(); - /* Remove it from the Timer List, add it to the Expired List */ - RemoveEntryList(&Timer->TimerListEntry); - InsertTailList(&ExpiredTimerList, &Timer->TimerListEntry); - NextEntry = ListHead->Flink; - } + /* Signal the timer if it's still on our list */ + if (!IsListEmpty(&ListHead)) RequestInterrupt = KiSignalTimer(Timer); - /* Expire the Timers */ - while (ExpiredTimerList.Flink != &ExpiredTimerList) - { - /* Get the Timer */ - Timer = CONTAINING_RECORD(ExpiredTimerList.Flink, - KTIMER, - TimerListEntry); - - /* Remove it */ - /// - // GCC IS A BRAINDEAD PIECE OF SHIT. WILL GIVE 5$ FOR EACH DEV KILLED. - /// - Timer->Header.Inserted = FALSE; - RemoveEntryList(&Timer->TimerListEntry); - //KiRemoveTimer(Timer); - - /* Expire it */ - KiHandleExpiredTimer(Timer); - } + /* Release the dispatcher lock */ + KiReleaseDispatcherLockFromDpcLevel(); - /* Release Dispatcher Lock */ - KiReleaseDispatcherLock(OldIrql); + /* Request a DPC if needed */ + if (RequestInterrupt) HalRequestSoftwareInterrupt(DISPATCH_LEVEL); } /* PUBLIC FUNCTIONS **********************************************************/ @@ -200,25 +254,19 @@ KeCancelTimer(IN OUT PKTIMER Timer) KIRQL OldIrql; BOOLEAN Inserted; ASSERT_TIMER(Timer); - ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL); + ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); /* Lock the Database and Raise IRQL */ OldIrql = KiAcquireDispatcherLock(); /* Check if it's inserted, and remove it if it is */ Inserted = Timer->Header.Inserted; - if (Inserted) - { - /// - // GCC IS A BRAINDEAD PIECE OF SHIT. WILL GIVE 5$ FOR EACH DEV KILLED. - /// - Timer->Header.Inserted = FALSE; - RemoveEntryList(&Timer->TimerListEntry); - //KiRemoveTimer(Timer); - } + if (Inserted) KxRemoveTreeTimer(Timer); /* Release Dispatcher Lock */ KiReleaseDispatcherLock(OldIrql); + + /* Return the old state */ return Inserted; } @@ -289,62 +337,75 @@ KeSetTimerEx(IN OUT PKTIMER Timer, { KIRQL OldIrql; BOOLEAN Inserted; + ULONG Hand = 0; + LARGE_INTEGER InterruptTime, SystemTime, DifferenceTime; + BOOLEAN RequestInterrupt = FALSE; ASSERT_TIMER(Timer); - ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL); + ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); /* Lock the Database and Raise IRQL */ OldIrql = KiAcquireDispatcherLock(); /* Check if it's inserted, and remove it if it is */ Inserted = Timer->Header.Inserted; - if (Inserted) - { - /// - // GCC IS A BRAINDEAD PIECE OF SHIT. WILL GIVE 5$ FOR EACH DEV KILLED. - /// - Timer->Header.Inserted = FALSE; - RemoveEntryList(&Timer->TimerListEntry); - //KiRemoveTimer(Timer); - } + if (Inserted) KxRemoveTreeTimer(Timer); /* Set Default Timer Data */ Timer->Dpc = Dpc; Timer->Period = Period; - Timer->Header.SignalState = FALSE; - /* Insert it */ - if (!KiInsertTimer(Timer, DueTime)) + /* Convert to relative time if needed */ + Timer->Header.Absolute = FALSE; + if (DueTime.HighPart >= 0) { - /* Check if it has any waiters */ - if (!IsListEmpty(&Timer->Header.WaitListHead)) - { - /* Wake them */ - KiWaitTest(Timer, IO_NO_INCREMENT); - } + /* Get System Time */ + KeQuerySystemTime(&SystemTime); - /* Check if the Timer has a DPC */ - if (Dpc) - { - /* Insert the DPC */ - KeInsertQueueDpc(Timer->Dpc, - NULL, - NULL); - } + /* Do the conversion */ + DifferenceTime.QuadPart = SystemTime.QuadPart - DueTime.QuadPart; - /* Check if the Timer is periodic */ - if (Timer->Period) + /* Make sure it hasn't already expired */ + Timer->Header.Absolute = TRUE; + if (DifferenceTime.HighPart >= 0) { - /* Reinsert the Timer */ - DueTime.QuadPart = Timer->Period * -SYSTEM_TIME_UNITS_PER_MSEC; - while (!KiInsertTimer(Timer, DueTime)); + /* Cancel everything */ + Timer->Header.SignalState = TRUE; + Timer->Header.Hand = 0; + Timer->DueTime.QuadPart = 0; + + /* Signal the timer */ + RequestInterrupt = KiSignalTimer(Timer); + + /* Release the dispatcher lock */ + KiReleaseDispatcherLockFromDpcLevel(); + + /* Check if we need to do an interrupt */ + if (RequestInterrupt) HalRequestSoftwareInterrupt(DISPATCH_LEVEL); } + + /* Set the time as Absolute */ + DueTime = DifferenceTime; } + /* Get the Interrupt Time */ + InterruptTime.QuadPart = KeQueryInterruptTime(); + + /* Recalculate due time */ + Timer->DueTime.QuadPart = InterruptTime.QuadPart - DueTime.QuadPart; + + /* Get the handle */ + Hand = KiComputeTimerTableIndex(Timer->DueTime.QuadPart); + Timer->Header.Hand = (UCHAR)Hand; + Timer->Header.Inserted = TRUE; + + /* Insert the timer */ + Timer->Header.SignalState = FALSE; + KxInsertTimer(Timer, Hand); + /* Release Dispatcher Lock */ - KiReleaseDispatcherLock(OldIrql); + KiExitDispatcher(OldIrql); /* Return old state */ return Inserted; } -/* EOF */ diff --git a/reactos/ntoskrnl/ke/wait.c b/reactos/ntoskrnl/ke/wait.c index 19ec2498323..dabe787ec23 100644 --- a/reactos/ntoskrnl/ke/wait.c +++ b/reactos/ntoskrnl/ke/wait.c @@ -77,13 +77,7 @@ KiUnlinkThread(IN PKTHREAD Thread, /* Check if there's a Thread Timer */ Timer = &Thread->Timer; - if (Timer->Header.Inserted) - { - /* Remove the timer */ - Timer->Header.Inserted = FALSE; - RemoveEntryList(&Timer->TimerListEntry); - //KiRemoveTimer(Timer); - } + if (Timer->Header.Inserted) KxRemoveTreeTimer(Timer); /* Increment the Queue's active threads */ if (Thread->Queue) Thread->Queue->CurrentCount++; @@ -219,6 +213,7 @@ KeDelayExecutionThread(IN KPROCESSOR_MODE WaitMode, BOOLEAN Swappable; PLARGE_INTEGER OriginalDueTime; LARGE_INTEGER DueTime, NewDueTime, InterruptTime; + ULONG Hand = 0; /* If this is a user-mode wait of 0 seconds, yield execution */ if (!(Interval->QuadPart) && (WaitMode != KernelMode)) @@ -285,7 +280,7 @@ KeDelayExecutionThread(IN KPROCESSOR_MODE WaitMode, /* Insert the timer and swap the thread */ ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL); KiSetThreadSwapBusy(Thread); - KiInsertWaitTimer(Timer); + KxInsertTimer(Timer, Hand); WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb()); /* Check if were swapped ok */ @@ -350,6 +345,7 @@ KeWaitForSingleObject(IN PVOID Object, BOOLEAN Swappable; LARGE_INTEGER DueTime, NewDueTime, InterruptTime; PLARGE_INTEGER OriginalDueTime = Timeout; + ULONG Hand = 0; /* Check if the lock is already held */ if (!Thread->WaitNext) goto WaitStart; @@ -449,7 +445,7 @@ KeWaitForSingleObject(IN PVOID Object, if (Timeout) { /* Insert it */ - KiInsertWaitTimer(Timer); + KxInsertTimer(Timer, Hand); } else { @@ -515,7 +511,7 @@ KeWaitForMultipleObjects(IN ULONG Count, BOOLEAN Swappable; PLARGE_INTEGER OriginalDueTime = Timeout; LARGE_INTEGER DueTime, NewDueTime, InterruptTime; - ULONG Index; + ULONG Index, Hand = 0; /* Make sure the Wait Count is valid */ if (!WaitBlockArray) @@ -726,7 +722,7 @@ KeWaitForMultipleObjects(IN ULONG Count, if (Timeout) { /* Insert it */ - KiInsertWaitTimer(Timer); + KxInsertTimer(Timer, Hand); } else { -- 2.17.1