#include <ntoskrnl.h>
#define NDEBUG
-#include <internal/debug.h>
-
-/* GLOBALS ******************************************************************/
-
-KSPIN_LOCK DispatcherDatabaseLock;
+#include <debug.h>
/* PRIVATE FUNCTIONS *********************************************************/
-VOID
-FASTCALL
-KiWaitSatisfyAll(PKWAIT_BLOCK FirstBlock)
-{
- PKWAIT_BLOCK WaitBlock = FirstBlock;
- PKTHREAD WaitThread = WaitBlock->Thread;
-
- /* Loop through all the Wait Blocks, and wake each Object */
- do
- {
- /* Make sure it hasn't timed out */
- if (WaitBlock->WaitKey != STATUS_TIMEOUT)
- {
- /* Wake the Object */
- KiSatisfyObjectWait((PKMUTANT)WaitBlock->Object, WaitThread);
- }
-
- /* Move to the next block */
- WaitBlock = WaitBlock->NextWaitBlock;
- } while (WaitBlock != FirstBlock);
-}
-
VOID
FASTCALL
KiWaitTest(IN PVOID ObjectPointer,
IN KPRIORITY Increment)
{
- PLIST_ENTRY WaitEntry;
- PLIST_ENTRY WaitList;
- PKWAIT_BLOCK CurrentWaitBlock;
- PKWAIT_BLOCK NextWaitBlock;
+ PLIST_ENTRY WaitEntry, WaitList;
+ PKWAIT_BLOCK WaitBlock;
PKTHREAD WaitThread;
- PKMUTANT FirstObject = ObjectPointer, Object;
+ PKMUTANT FirstObject = ObjectPointer;
+ NTSTATUS WaitStatus;
/* Loop the Wait Entries */
WaitList = &FirstObject->Header.WaitListHead;
WaitEntry = WaitList->Flink;
- while ((FirstObject->Header.SignalState > 0) &&
- (WaitEntry != WaitList))
+ while ((FirstObject->Header.SignalState > 0) && (WaitEntry != WaitList))
{
/* Get the current wait block */
- CurrentWaitBlock = CONTAINING_RECORD(WaitEntry,
- KWAIT_BLOCK,
- WaitListEntry);
- WaitThread = CurrentWaitBlock->Thread;
+ WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
+ WaitThread = WaitBlock->Thread;
+ WaitStatus = STATUS_KERNEL_APC;
/* Check the current Wait Mode */
- if (CurrentWaitBlock->WaitType == WaitAny)
+ if (WaitBlock->WaitType == WaitAny)
{
/* Easy case, satisfy only this wait */
- WaitEntry = WaitEntry->Blink;
+ WaitStatus = (NTSTATUS)WaitBlock->WaitKey;
KiSatisfyObjectWait(FirstObject, WaitThread);
}
- else
- {
- /* Everything must be satisfied */
- NextWaitBlock = CurrentWaitBlock->NextWaitBlock;
-
- /* Loop first to make sure they are valid */
- while (NextWaitBlock != CurrentWaitBlock)
- {
- /* Make sure this isn't a timeout block */
- if (NextWaitBlock->WaitKey != STATUS_TIMEOUT)
- {
- /* Get the object */
- Object = NextWaitBlock->Object;
-
- /* Check if this is a mutant */
- if ((Object->Header.Type == MutantObject) &&
- (Object->Header.SignalState <= 0) &&
- (WaitThread == Object->OwnerThread))
- {
- /* It's a signaled mutant */
- }
- else if (Object->Header.SignalState <= 0)
- {
- /* Skip the unwaiting */
- goto SkipUnwait;
- }
- }
-
- /* Go to the next Wait block */
- NextWaitBlock = NextWaitBlock->NextWaitBlock;
- }
-
- /* All the objects are signaled, we can satisfy */
- WaitEntry = WaitEntry->Blink;
- KiWaitSatisfyAll(CurrentWaitBlock);
- }
-
- /* All waits satisfied, unwait the thread */
- KiAbortWaitThread(WaitThread, CurrentWaitBlock->WaitKey, Increment);
-SkipUnwait:
- /* Next entry */
- WaitEntry = WaitEntry->Flink;
+ /* Now do the rest of the unwait */
+ KiUnwaitThread(WaitThread, WaitStatus, Increment);
+ WaitEntry = WaitList->Flink;
}
}
-/* Must be called with the dispatcher lock held */
VOID
FASTCALL
-KiAbortWaitThread(IN PKTHREAD Thread,
- IN NTSTATUS WaitStatus,
- IN KPRIORITY Increment)
+KiUnlinkThread(IN PKTHREAD Thread,
+ IN NTSTATUS WaitStatus)
{
PKWAIT_BLOCK WaitBlock;
PKTIMER Timer;
- LONG NewPriority;
/* Update wait status */
Thread->WaitStatus |= WaitStatus;
/* Check if there's a Thread Timer */
Timer = &Thread->Timer;
- if (Timer->Header.Inserted)
- {
- /* Remove the timer */
- Timer->Header.Inserted = FALSE;
- RemoveEntryList(&Timer->TimerListEntry);
- //KiRemoveTimer(Timer);
- }
+ if (Timer->Header.Inserted) KxRemoveTreeTimer(Timer);
/* Increment the Queue's active threads */
if (Thread->Queue) Thread->Queue->CurrentCount++;
+}
- /* Check if this is a non-RT thread */
- if (Thread->Priority < LOW_REALTIME_PRIORITY)
- {
- /* Check if boosting is enabled and we can boost */
- if (!(Thread->DisableBoost) && !(Thread->PriorityDecrement))
- {
- /* We can boost, so calculate the new priority */
- NewPriority = Thread->BasePriority + Increment;
- if (NewPriority > Thread->Priority)
- {
- /* Make sure the new priority wouldn't push the thread to RT */
- if (NewPriority >= LOW_REALTIME_PRIORITY)
- {
- /* Set it just before the RT zone */
- Thread->Priority = LOW_REALTIME_PRIORITY - 1;
- }
- else
- {
- /* Otherwise, set our calculated priority */
- Thread->Priority = NewPriority;
- }
- }
- }
-
- /* Check if this is a high-priority thread */
- if (Thread->BasePriority >= 14)
- {
- /* It is, simply reset the quantum */
- Thread->Quantum = Thread->QuantumReset;
- }
- else
- {
- /* Otherwise, decrease quantum */
- Thread->Quantum--;
- if (Thread->Quantum <= 0)
- {
- /* We've went below 0, reset it */
- Thread->Quantum = Thread->QuantumReset;
-
- /* Apply per-quantum priority decrement */
- Thread->Priority -= (Thread->PriorityDecrement + 1);
- if (Thread->Priority < Thread->BasePriority)
- {
- /* We've went too low, reset it */
- Thread->Priority = Thread->BasePriority;
- }
+/* Must be called with the dispatcher lock held */
+VOID
+FASTCALL
+KiUnwaitThread(IN PKTHREAD Thread,
+ IN LONG_PTR WaitStatus,
+ IN KPRIORITY Increment)
+{
+ /* Unlink the thread */
+ KiUnlinkThread(Thread, WaitStatus);
- /* Delete per-quantum decrement */
- Thread->PriorityDecrement = 0;
- }
- }
- }
- else
- {
- /* For real time threads, just reset the quantum */
- Thread->Quantum = Thread->QuantumReset;
- }
+ /* Tell the scheduler do to the increment when it readies the thread */
+ ASSERT(Increment >= 0);
+ Thread->AdjustIncrement = (SCHAR)Increment;
+ Thread->AdjustReason = AdjustUnwait;
/* Reschedule the Thread */
KiReadyThread(Thread);
FastMutex->Contention++;
/* Wait for the event */
- KeWaitForSingleObject(&FastMutex->Gate,
+ KeWaitForSingleObject(&FastMutex->Event,
WrMutex,
KernelMode,
FALSE,
NULL);
}
+VOID
+FASTCALL
+KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ ULONG BitsToRemove, BitsToAdd;
+ LONG OldValue, NewValue;
+
+ /* We depend on these bits being just right */
+ C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC);
+
+ /* Increase the contention count */
+ GuardedMutex->Contention++;
+
+ /* Start by unlocking the Guarded Mutex */
+ BitsToRemove = GM_LOCK_BIT;
+ BitsToAdd = GM_LOCK_WAITER_INC;
+
+ /* Start change loop */
+ for (;;)
+ {
+ /* Loop sanity checks */
+ ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+ (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN)));
+ ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) ||
+ (BitsToAdd == GM_LOCK_WAITER_WOKEN));
+
+ /* Get the Count Bits */
+ OldValue = GuardedMutex->Count;
+
+ /* Start internal bit change loop */
+ for (;;)
+ {
+ /* Check if the Guarded Mutex is locked */
+ if (OldValue & GM_LOCK_BIT)
+ {
+ /* Sanity check */
+ ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+ ((OldValue & GM_LOCK_WAITER_WOKEN) != 0));
+
+ /* Unlock it by removing the Lock Bit */
+ NewValue = OldValue ^ BitsToRemove;
+ NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+ NewValue,
+ OldValue);
+ if (NewValue == OldValue) return;
+ }
+ else
+ {
+ /* The Guarded Mutex isn't locked, so simply set the bits */
+ NewValue = OldValue + BitsToAdd;
+ NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+ NewValue,
+ OldValue);
+ if (NewValue == OldValue) break;
+ }
+
+ /* Old value changed, loop again */
+ OldValue = NewValue;
+ }
+
+ /* Now we have to wait for it */
+ KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode);
+ ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0);
+
+ /* Ok, the wait is done, so set the new bits */
+ BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN;
+ BitsToAdd = GM_LOCK_WAITER_WOKEN;
+ }
+}
+
//
// This routine exits the dispatcher after a compatible operation and
// swaps the context to the next scheduled thread on the current CPU if
BOOLEAN PendingApc;
/* Make sure we're at synchronization level */
- ASSERT_IRQL(SYNCH_LEVEL);
+ ASSERT(KeGetCurrentIrql() == SYNCH_LEVEL);
/* Check if we have deferred threads */
KiCheckDeferredReadyList(Prcb);
/* Make sure there's a new thread scheduled */
if (!Prcb->NextThread) goto Quickie;
- /* This shouldn't happen on ROS yet */
- DPRINT1("The impossible happened - Tell Alex\n");
- ASSERT(FALSE);
-
/* Lock the PRCB */
KiAcquirePrcbLock(Prcb);
IN BOOLEAN Alertable,
IN PLARGE_INTEGER Interval OPTIONAL)
{
- PKWAIT_BLOCK TimerWaitBlock;
- PKTIMER ThreadTimer;
- PKTHREAD CurrentThread = KeGetCurrentThread();
- NTSTATUS WaitStatus = STATUS_SUCCESS;
+ PKTIMER Timer;
+ PKWAIT_BLOCK TimerBlock;
+ PKTHREAD Thread = KeGetCurrentThread();
+ NTSTATUS WaitStatus;
BOOLEAN Swappable;
- PLARGE_INTEGER OriginalDueTime = Interval;
- LARGE_INTEGER DueTime, NewDueTime;
+ PLARGE_INTEGER OriginalDueTime;
+ LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
+ ULONG Hand = 0;
- /* Check if the lock is already held */
- if (CurrentThread->WaitNext)
+ /* If this is a user-mode wait of 0 seconds, yield execution */
+ if (!(Interval->QuadPart) && (WaitMode != KernelMode))
{
- /* Lock is held, disable Wait Next */
- CurrentThread->WaitNext = FALSE;
- }
- else
- {
- /* Lock not held, acquire it */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
+ /* Make sure the wait isn't alertable or interrupting an APC */
+ if (!(Alertable) && !(Thread->ApcState.UserApcPending))
+ {
+ /* Yield execution */
+ NtYieldExecution();
+ }
}
- /* Use built-in Wait block */
- TimerWaitBlock = &CurrentThread->WaitBlock[TIMER_WAIT_BLOCK];
+ /* Setup the original time and timer/wait blocks */
+ OriginalDueTime = Interval;
+ Timer = &Thread->Timer;
+ TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
- /* Start Wait Loop */
- do
+ /* Check if the lock is already held */
+ if (!Thread->WaitNext) goto WaitStart;
+
+ /* Otherwise, we already have the lock, so initialize the wait */
+ Thread->WaitNext = FALSE;
+ KxDelayThreadWait();
+
+ /* Start wait loop */
+ for (;;)
{
+ /* Disable pre-emption */
+ Thread->Preempted = FALSE;
+
/* Check if a kernel APC is pending and we're below APC_LEVEL */
- if ((CurrentThread->ApcState.KernelApcPending) &&
- !(CurrentThread->SpecialApcDisable) &&
- (CurrentThread->WaitIrql < APC_LEVEL))
+ if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+ (Thread->WaitIrql < APC_LEVEL))
{
/* Unlock the dispatcher */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ KiReleaseDispatcherLock(Thread->WaitIrql);
}
else
{
- /* Check if we can do an alertable wait, if requested */
- KiCheckAlertability();
-
- /* Check if we can swap the thread's stack */
- CurrentThread->WaitListEntry.Flink = NULL;
- KiCheckThreadStackSwap(WaitMode, CurrentThread, Swappable);
-
- /* Set status */
- CurrentThread->WaitStatus = STATUS_WAIT_0;
+ /* Check if we have to bail out due to an alerted state */
+ WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+ if (WaitStatus != STATUS_WAIT_0) break;
- /* Set Timer */
- ThreadTimer = &CurrentThread->Timer;
-
- /* Setup the Wait Block */
- CurrentThread->WaitBlockList = TimerWaitBlock;
- TimerWaitBlock->NextWaitBlock = TimerWaitBlock;
-
- /* Link the timer to this Wait Block */
- ThreadTimer->Header.WaitListHead.Flink =
- &TimerWaitBlock->WaitListEntry;
- ThreadTimer->Header.WaitListHead.Blink =
- &TimerWaitBlock->WaitListEntry;
-
- /* Insert the Timer into the Timer Lists and enable it */
- if (!KiInsertTimer(ThreadTimer, *Interval))
+ /* Check if the timer expired */
+ InterruptTime.QuadPart = KeQueryInterruptTime();
+ if ((ULONGLONG)InterruptTime.QuadPart >= Timer->DueTime.QuadPart)
{
- /* FIXME: We should find a new ready thread */
- WaitStatus = STATUS_SUCCESS;
- break;
+ /* It did, so we don't need to wait */
+ goto NoWait;
}
- /* Save due time */
- DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
+ /* It didn't, so activate it */
+ Timer->Header.Inserted = TRUE;
/* Handle Kernel Queues */
- if (CurrentThread->Queue) KiWakeQueue(CurrentThread->Queue);
+ if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
/* Setup the wait information */
- CurrentThread->Alertable = Alertable;
- CurrentThread->WaitMode = WaitMode;
- CurrentThread->WaitReason = DelayExecution;
- CurrentThread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
- CurrentThread->State = Waiting;
-
- /* Find a new thread to run */
- KiAddThreadToWaitList(CurrentThread, Swappable);
- WaitStatus = KiSwapThread(CurrentThread, KeGetCurrentPrcb());
- ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
-
- /* Check if we were executing an APC or if we timed out */
+ Thread->State = Waiting;
+
+ /* Add the thread to the wait list */
+ KiAddThreadToWaitList(Thread, Swappable);
+
+ /* Insert the timer and swap the thread */
+ ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+ KiSetThreadSwapBusy(Thread);
+ KxInsertTimer(Timer, Hand);
+ WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
+
+ /* Check if were swapped ok */
if (WaitStatus != STATUS_KERNEL_APC)
{
/* This is a good thing */
&NewDueTime);
}
- /* Acquire again the lock */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
- } while (TRUE);
+WaitStart:
+ /* Setup a new wait */
+ Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+ KxDelayThreadWait();
+ KiAcquireDispatcherLockAtDpcLevel();
+ }
- /* Release the Lock, we are done */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ /* We're done! */
+ KiReleaseDispatcherLock(Thread->WaitIrql);
return WaitStatus;
+
+NoWait:
+ /* There was nothing to wait for. Did we have a wait interval? */
+ if (!Interval->QuadPart)
+ {
+ /* Unlock the dispatcher and do a yield */
+ KiReleaseDispatcherLock(Thread->WaitIrql);
+ return NtYieldExecution();
+ }
+
+ /* Unlock the dispatcher and adjust the quantum for a no-wait */
+ KiReleaseDispatcherLockFromDpcLevel();
+ KiAdjustQuantumThread(Thread);
+ return STATUS_SUCCESS;
}
/*
IN BOOLEAN Alertable,
IN PLARGE_INTEGER Timeout OPTIONAL)
{
- PKMUTANT CurrentObject;
- PKWAIT_BLOCK WaitBlock;
- PKWAIT_BLOCK TimerWaitBlock;
- PKTIMER ThreadTimer;
- PKTHREAD CurrentThread = KeGetCurrentThread();
- NTSTATUS WaitStatus = STATUS_SUCCESS;
+ PKTHREAD Thread = KeGetCurrentThread();
+ PKMUTANT CurrentObject = (PKMUTANT)Object;
+ PKWAIT_BLOCK WaitBlock = &Thread->WaitBlock[0];
+ PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+ PKTIMER Timer = &Thread->Timer;
+ NTSTATUS WaitStatus;
BOOLEAN Swappable;
- LARGE_INTEGER DueTime, NewDueTime;
+ LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
PLARGE_INTEGER OriginalDueTime = Timeout;
+ ULONG Hand = 0;
/* Check if the lock is already held */
- if (CurrentThread->WaitNext)
- {
- /* Lock is held, disable Wait Next */
- CurrentThread->WaitNext = FALSE;
- }
- else
- {
- /* Lock not held, acquire it */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
- }
+ if (!Thread->WaitNext) goto WaitStart;
- /* Start the actual Loop */
- WaitBlock = &CurrentThread->WaitBlock[0];
- do
+ /* Otherwise, we already have the lock, so initialize the wait */
+ Thread->WaitNext = FALSE;
+ KxSingleThreadWait();
+
+ /* Start wait loop */
+ for (;;)
{
+ /* Disable pre-emption */
+ Thread->Preempted = FALSE;
+
/* Check if a kernel APC is pending and we're below APC_LEVEL */
- if ((CurrentThread->ApcState.KernelApcPending) &&
- !(CurrentThread->SpecialApcDisable) &&
- (CurrentThread->WaitIrql < APC_LEVEL))
+ if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+ (Thread->WaitIrql < APC_LEVEL))
{
/* Unlock the dispatcher */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ KiReleaseDispatcherLock(Thread->WaitIrql);
}
else
{
- /* Set default status */
- CurrentThread->WaitStatus = STATUS_WAIT_0;
-
- /* Get the Current Object */
- CurrentObject = (PKMUTANT)Object;
+ /* Sanity check */
+ ASSERT(CurrentObject->Header.Type != QueueObject);
/* Check if it's a mutant */
if (CurrentObject->Header.Type == MutantObject)
{
/* Check its signal state or if we own it */
if ((CurrentObject->Header.SignalState > 0) ||
- (CurrentThread == CurrentObject->OwnerThread))
+ (Thread == CurrentObject->OwnerThread))
{
/* Just unwait this guy and exit */
if (CurrentObject->Header.SignalState != (LONG)MINLONG)
{
/* It has a normal signal state. Unwait and return */
- KiSatisfyMutantWait(CurrentObject, CurrentThread);
- WaitStatus = CurrentThread->WaitStatus;
+ KiSatisfyMutantWait(CurrentObject, Thread);
+ WaitStatus = Thread->WaitStatus;
goto DontWait;
}
else
{
- /* Raise an exception (see wasm.ru) */
- KiReleaseDispatcherLock(CurrentThread->
- WaitIrql);
+ /* Raise an exception */
+ KiReleaseDispatcherLock(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
}
}
else if (CurrentObject->Header.SignalState > 0)
{
/* Another satisfied object */
- KiSatisfyNonMutantWait(CurrentObject, CurrentThread);
+ KiSatisfyNonMutantWait(CurrentObject);
WaitStatus = STATUS_WAIT_0;
goto DontWait;
}
- /* Append wait block to the KTHREAD wait block list */
- CurrentThread->WaitBlockList = WaitBlock;
-
- /* Set up the Wait Block */
- WaitBlock->Object = CurrentObject;
- WaitBlock->WaitKey = (USHORT)(STATUS_SUCCESS);
- WaitBlock->WaitType = WaitAny;
-
/* Make sure we can satisfy the Alertable request */
- KiCheckAlertability();
-
- /* Check if we can swap the thread's stack */
- CurrentThread->WaitListEntry.Flink = NULL;
- KiCheckThreadStackSwap(WaitMode, CurrentThread, Swappable);
+ WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+ if (WaitStatus != STATUS_WAIT_0) break;
/* Enable the Timeout Timer if there was any specified */
if (Timeout)
{
- /* Fail if the timeout interval is actually 0 */
- if (!Timeout->QuadPart)
- {
- /* Return a timeout */
- WaitStatus = STATUS_TIMEOUT;
- goto DontWait;
- }
-
- /* Point to Timer Wait Block and Thread Timer */
- TimerWaitBlock = &CurrentThread->WaitBlock[TIMER_WAIT_BLOCK];
- ThreadTimer = &CurrentThread->Timer;
-
- /* Connect the Timer Wait Block */
- WaitBlock->NextWaitBlock = TimerWaitBlock;
-
- /* Set up the Timer Wait Block */
- TimerWaitBlock->NextWaitBlock = WaitBlock;
-
- /* Link the timer to this Wait Block */
- ThreadTimer->Header.WaitListHead.Flink =
- &TimerWaitBlock->WaitListEntry;
- ThreadTimer->Header.WaitListHead.Blink =
- &TimerWaitBlock->WaitListEntry;
-
- /* Insert the Timer into the Timer Lists and enable it */
- if (!KiInsertTimer(ThreadTimer, *Timeout))
+ /* Check if the timer expired */
+ InterruptTime.QuadPart = KeQueryInterruptTime();
+ if ((ULONGLONG)InterruptTime.QuadPart >=
+ Timer->DueTime.QuadPart)
{
- /* Return a timeout if we couldn't insert the timer */
+ /* It did, so we don't need to wait */
WaitStatus = STATUS_TIMEOUT;
goto DontWait;
}
- /* Set the current due time */
- DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
- }
- else
- {
- /* No timer block, so just set our wait block as next */
- WaitBlock->NextWaitBlock = WaitBlock;
+ /* It didn't, so activate it */
+ Timer->Header.Inserted = TRUE;
}
/* Link the Object to this Wait Block */
&WaitBlock->WaitListEntry);
/* Handle Kernel Queues */
- if (CurrentThread->Queue) KiWakeQueue(CurrentThread->Queue);
+ if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
/* Setup the wait information */
- CurrentThread->Alertable = Alertable;
- CurrentThread->WaitMode = WaitMode;
- CurrentThread->WaitReason = WaitReason;
- CurrentThread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
- CurrentThread->State = Waiting;
+ Thread->State = Waiting;
- /* Find a new thread to run */
- KiAddThreadToWaitList(CurrentThread, Swappable);
- WaitStatus = KiSwapThread(CurrentThread, KeGetCurrentPrcb());
- ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
+ /* Add the thread to the wait list */
+ KiAddThreadToWaitList(Thread, Swappable);
+
+ /* Activate thread swap */
+ ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+ KiSetThreadSwapBusy(Thread);
+
+ /* Check if we have a timer */
+ if (Timeout)
+ {
+ /* Insert it */
+ KxInsertTimer(Timer, Hand);
+ }
+ else
+ {
+ /* Otherwise, unlock the dispatcher */
+ KiReleaseDispatcherLockFromDpcLevel();
+ }
+
+ /* Do the actual swap */
+ WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
/* Check if we were executing an APC */
if (WaitStatus != STATUS_KERNEL_APC) return WaitStatus;
&NewDueTime);
}
}
+WaitStart:
+ /* Setup a new wait */
+ Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+ KxSingleThreadWait();
+ KiAcquireDispatcherLockAtDpcLevel();
+ }
- /* Acquire again the lock */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
- } while (TRUE);
-
- /* Release the Lock, we are done */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ /* Wait complete */
+ KiReleaseDispatcherLock(Thread->WaitIrql);
return WaitStatus;
DontWait:
- /* Adjust the Quantum */
- KiAdjustQuantumThread(CurrentThread);
+ /* Release dispatcher lock but maintain high IRQL */
+ KiReleaseDispatcherLockFromDpcLevel();
- /* Release & Return */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ /* Adjust the Quantum and return the wait status */
+ KiAdjustQuantumThread(Thread);
return WaitStatus;
}
{
PKMUTANT CurrentObject;
PKWAIT_BLOCK WaitBlock;
- PKWAIT_BLOCK TimerWaitBlock;
- PKTIMER ThreadTimer;
- PKTHREAD CurrentThread = KeGetCurrentThread();
- ULONG AllObjectsSignaled;
- ULONG WaitIndex;
+ PKTHREAD Thread = KeGetCurrentThread();
+ PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+ PKTIMER Timer = &Thread->Timer;
NTSTATUS WaitStatus = STATUS_SUCCESS;
BOOLEAN Swappable;
PLARGE_INTEGER OriginalDueTime = Timeout;
- LARGE_INTEGER DueTime, NewDueTime;
-
- /* Set the Current Thread */
- CurrentThread = KeGetCurrentThread();
-
- /* Check if the lock is already held */
- if (CurrentThread->WaitNext)
- {
- /* Lock is held, disable Wait Next */
- CurrentThread->WaitNext = FALSE;
- }
- else
- {
- /* Lock not held, acquire it */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
- }
+ LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
+ ULONG Index, Hand = 0;
/* Make sure the Wait Count is valid */
if (!WaitBlockArray)
if (Count > THREAD_WAIT_OBJECTS)
{
/* Bugcheck */
- KEBUGCHECK(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
+ KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
}
/* Use the Thread's Wait Block */
- WaitBlockArray = &CurrentThread->WaitBlock[0];
+ WaitBlockArray = &Thread->WaitBlock[0];
}
else
{
if (Count > MAXIMUM_WAIT_OBJECTS)
{
/* Bugcheck */
- KEBUGCHECK(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
+ KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
}
}
- /* Start the actual Loop */
- do
+ /* Sanity check */
+ ASSERT(Count != 0);
+
+ /* Check if the lock is already held */
+ if (!Thread->WaitNext) goto WaitStart;
+
+ /* Otherwise, we already have the lock, so initialize the wait */
+ Thread->WaitNext = FALSE;
+ /* Note that KxMultiThreadWait is a macro, defined in ke_x.h, that */
+ /* uses (and modifies some of) the following local */
+ /* variables: */
+ /* Thread, Index, WaitBlock, Timer, Timeout, Hand and Swappable. */
+ /* If it looks like this code doesn't actually wait for any objects */
+ /* at all, it's because the setup is done by that macro. */
+ KxMultiThreadWait();
+
+ /* Start wait loop */
+ for (;;)
{
+ /* Disable pre-emption */
+ Thread->Preempted = FALSE;
+
/* Check if a kernel APC is pending and we're below APC_LEVEL */
- if ((CurrentThread->ApcState.KernelApcPending) &&
- !(CurrentThread->SpecialApcDisable) &&
- (CurrentThread->WaitIrql < APC_LEVEL))
+ if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+ (Thread->WaitIrql < APC_LEVEL))
{
/* Unlock the dispatcher */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ KiReleaseDispatcherLock(Thread->WaitIrql);
}
else
{
- /* Append wait block to the KTHREAD wait block list */
- CurrentThread->WaitBlockList = WaitBlock = WaitBlockArray;
-
- /* Set default wait status */
- CurrentThread->WaitStatus = STATUS_WAIT_0;
-
- /* Check if the wait is (already) satisfied */
- AllObjectsSignaled = TRUE;
-
- /* First, we'll try to satisfy the wait directly */
- for (WaitIndex = 0; WaitIndex < Count; WaitIndex++)
+ /* Check what kind of wait this is */
+ Index = 0;
+ if (WaitType == WaitAny)
{
- /* Get the Current Object */
- CurrentObject = (PKMUTANT)Object[WaitIndex];
-
- /* Check the type of wait */
- if (WaitType == WaitAny)
+ /* Loop blocks */
+ do
{
+ /* Get the Current Object */
+ CurrentObject = (PKMUTANT)Object[Index];
+ ASSERT(CurrentObject->Header.Type != QueueObject);
+
/* Check if the Object is a mutant */
if (CurrentObject->Header.Type == MutantObject)
{
/* Check if it's signaled */
if ((CurrentObject->Header.SignalState > 0) ||
- (CurrentThread == CurrentObject->OwnerThread))
+ (Thread == CurrentObject->OwnerThread))
{
/* This is a Wait Any, so unwait this and exit */
if (CurrentObject->Header.SignalState !=
(LONG)MINLONG)
{
/* Normal signal state, unwait it and return */
- KiSatisfyMutantWait(CurrentObject,
- CurrentThread);
- WaitStatus = CurrentThread->WaitStatus |
- WaitIndex;
+ KiSatisfyMutantWait(CurrentObject, Thread);
+ WaitStatus = Thread->WaitStatus | Index;
goto DontWait;
}
else
{
/* Raise an exception (see wasm.ru) */
- KiReleaseDispatcherLock(CurrentThread->
- WaitIrql);
+ KiReleaseDispatcherLock(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
}
}
else if (CurrentObject->Header.SignalState > 0)
{
/* Another signaled object, unwait and return */
- KiSatisfyNonMutantWait(CurrentObject, CurrentThread);
- WaitStatus = WaitIndex;
+ KiSatisfyNonMutantWait(CurrentObject);
+ WaitStatus = Index;
goto DontWait;
}
- }
- else
+
+ /* Go to the next block */
+ Index++;
+ } while (Index < Count);
+ }
+ else
+ {
+ /* Loop blocks */
+ do
{
+ /* Get the Current Object */
+ CurrentObject = (PKMUTANT)Object[Index];
+ ASSERT(CurrentObject->Header.Type != QueueObject);
+
/* Check if we're dealing with a mutant again */
if (CurrentObject->Header.Type == MutantObject)
{
/* Check if it has an invalid count */
- if ((CurrentThread == CurrentObject->OwnerThread) &&
- (CurrentObject->Header.SignalState == MINLONG))
+ if ((Thread == CurrentObject->OwnerThread) &&
+ (CurrentObject->Header.SignalState == (LONG)MINLONG))
{
/* Raise an exception */
- KiReleaseDispatcherLock(CurrentThread->
- WaitIrql);
+ KiReleaseDispatcherLock(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
}
else if ((CurrentObject->Header.SignalState <= 0) &&
- (CurrentThread != CurrentObject->OwnerThread))
+ (Thread != CurrentObject->OwnerThread))
{
/* We don't own it, can't satisfy the wait */
- AllObjectsSignaled = FALSE;
+ break;
}
}
else if (CurrentObject->Header.SignalState <= 0)
{
/* Not signaled, can't satisfy */
- AllObjectsSignaled = FALSE;
+ break;
}
- }
-
- /* Set up a Wait Block for this Object */
- WaitBlock->Object = CurrentObject;
- WaitBlock->Thread = CurrentThread;
- WaitBlock->WaitKey = (USHORT)WaitIndex;
- WaitBlock->WaitType = (USHORT)WaitType;
- WaitBlock->NextWaitBlock = WaitBlock + 1;
- /* Move to the next Wait Block */
- WaitBlock = WaitBlock->NextWaitBlock;
- }
+ /* Go to the next block */
+ Index++;
+ } while (Index < Count);
- /* Return to the Root Wait Block */
- WaitBlock--;
- WaitBlock->NextWaitBlock = WaitBlockArray;
+ /* Check if we've went through all the objects */
+ if (Index == Count)
+ {
+ /* Loop wait blocks */
+ WaitBlock = WaitBlockArray;
+ do
+ {
+ /* Get the object and satisfy it */
+ CurrentObject = (PKMUTANT)WaitBlock->Object;
+ KiSatisfyObjectWait(CurrentObject, Thread);
- /* Check if this is a Wait All and all the objects are signaled */
- if ((WaitType == WaitAll) && (AllObjectsSignaled))
- {
- /* Return to the Root Wait Block */
- WaitBlock = CurrentThread->WaitBlockList;
+ /* Go to the next block */
+ WaitBlock = WaitBlock->NextWaitBlock;
+ } while(WaitBlock != WaitBlockArray);
- /* Satisfy their Waits and return to the caller */
- KiWaitSatisfyAll(WaitBlock);
- WaitStatus = CurrentThread->WaitStatus;
- goto DontWait;
+ /* Set the wait status and get out */
+ WaitStatus = Thread->WaitStatus;
+ goto DontWait;
+ }
}
/* Make sure we can satisfy the Alertable request */
- KiCheckAlertability();
-
- /* Check if we can swap the thread's stack */
- CurrentThread->WaitListEntry.Flink = NULL;
- KiCheckThreadStackSwap(WaitMode, CurrentThread, Swappable);
+ WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+ if (WaitStatus != STATUS_WAIT_0) break;
/* Enable the Timeout Timer if there was any specified */
if (Timeout)
{
- /* Make sure the timeout interval isn't actually 0 */
- if (!Timeout->QuadPart)
+ /* Check if the timer expired */
+ InterruptTime.QuadPart = KeQueryInterruptTime();
+ if ((ULONGLONG)InterruptTime.QuadPart >=
+ Timer->DueTime.QuadPart)
{
- /* Return a timeout */
+ /* It did, so we don't need to wait */
WaitStatus = STATUS_TIMEOUT;
goto DontWait;
}
- /* Point to Timer Wait Block and Thread Timer */
- TimerWaitBlock = &CurrentThread->WaitBlock[TIMER_WAIT_BLOCK];
- ThreadTimer = &CurrentThread->Timer;
-
- /* Connect the Timer Wait Block */
- WaitBlock->NextWaitBlock = TimerWaitBlock;
-
- /* Set up the Timer Wait Block */
- TimerWaitBlock->NextWaitBlock = WaitBlockArray;
-
- /* Initialize the list head */
- InitializeListHead(&ThreadTimer->Header.WaitListHead);
+ /* It didn't, so activate it */
+ Timer->Header.Inserted = TRUE;
- /* Insert the Timer into the Timer Lists and enable it */
- if (!KiInsertTimer(ThreadTimer, *Timeout))
- {
- /* Return a timeout if we couldn't insert the timer */
- WaitStatus = STATUS_TIMEOUT;
- goto DontWait;
- }
-
- /* Set the current due time */
- DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
+ /* Link the wait blocks */
+ WaitBlock->NextWaitBlock = TimerBlock;
}
/* Insert into Object's Wait List*/
- WaitBlock = CurrentThread->WaitBlockList;
+ WaitBlock = WaitBlockArray;
do
{
/* Get the Current Object */
} while (WaitBlock != WaitBlockArray);
/* Handle Kernel Queues */
- if (CurrentThread->Queue) KiWakeQueue(CurrentThread->Queue);
+ if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
/* Setup the wait information */
- CurrentThread->Alertable = Alertable;
- CurrentThread->WaitMode = WaitMode;
- CurrentThread->WaitReason = WaitReason;
- CurrentThread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
- CurrentThread->State = Waiting;
+ Thread->State = Waiting;
+
+ /* Add the thread to the wait list */
+ KiAddThreadToWaitList(Thread, Swappable);
- /* Find a new thread to run */
- KiAddThreadToWaitList(CurrentThread, Swappable);
- WaitStatus = KiSwapThread(CurrentThread, KeGetCurrentPrcb());
- ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
+ /* Activate thread swap */
+ ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+ KiSetThreadSwapBusy(Thread);
+
+ /* Check if we have a timer */
+ if (Timeout)
+ {
+ /* Insert it */
+ KxInsertTimer(Timer, Hand);
+ }
+ else
+ {
+ /* Otherwise, unlock the dispatcher */
+ KiReleaseDispatcherLockFromDpcLevel();
+ }
+
+ /* Swap the thread */
+ WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
/* Check if we were executing an APC */
if (WaitStatus != STATUS_KERNEL_APC) return WaitStatus;
&DueTime,
&NewDueTime);
}
-
- /* Acquire again the lock */
- CurrentThread->WaitIrql = KiAcquireDispatcherLock();
}
- } while (TRUE);
- /* Release the Lock, we are done */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+WaitStart:
+ /* Setup a new wait */
+ Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+ KxMultiThreadWait();
+ KiAcquireDispatcherLockAtDpcLevel();
+ }
+
+ /* We are done */
+ KiReleaseDispatcherLock(Thread->WaitIrql);
return WaitStatus;
DontWait:
- /* Adjust the Quantum */
- KiAdjustQuantumThread(CurrentThread);
+ /* Release dispatcher lock but maintain high IRQL */
+ KiReleaseDispatcherLockFromDpcLevel();
- /* Release & Return */
- KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+ /* Adjust the Quantum and return the wait status */
+ KiAdjustQuantumThread(Thread);
return WaitStatus;
}
{
KPROCESSOR_MODE PreviousMode = ExGetPreviousMode();
LARGE_INTEGER SafeInterval;
- NTSTATUS Status = STATUS_SUCCESS;
+ NTSTATUS Status;
- /* Check if parameters are valid */
- if(PreviousMode != KernelMode)
+ /* Check the previous mode */
+ if (PreviousMode != KernelMode)
{
- _SEH_TRY
+ /* Enter SEH for probing */
+ _SEH2_TRY
{
- /* make a copy on the kernel stack and let DelayInterval point to it so
- we don't need to wrap KeDelayExecutionThread in SEH! */
+ /* Probe and capture the time out */
SafeInterval = ProbeForReadLargeInteger(DelayInterval);
DelayInterval = &SafeInterval;
}
- _SEH_HANDLE
+ _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
{
- Status = _SEH_GetExceptionCode();
+ /* Return the exception code */
+ _SEH2_YIELD(return _SEH2_GetExceptionCode());
}
- _SEH_END;
-
- if (!NT_SUCCESS(Status)) return Status;
+ _SEH2_END;
}
/* Call the Kernel Function */
projects / reactos.git / blobdiff