[HEADERS]
[reactos.git] / reactos / ntoskrnl / ke / wait.c
index 21bf975..a4737b0 100644 (file)
 
 #include <ntoskrnl.h>
 #define NDEBUG
-#include <internal/debug.h>
+#include <debug.h>
 
 /* PRIVATE FUNCTIONS *********************************************************/
 
-VOID
-FASTCALL
-KiWaitSatisfyAll(PKWAIT_BLOCK FirstBlock)
-{
-    PKWAIT_BLOCK WaitBlock = FirstBlock;
-    PKTHREAD WaitThread = WaitBlock->Thread;
-
-    /* Loop through all the Wait Blocks, and wake each Object */
-    do
-    {
-        /* Make sure it hasn't timed out */
-        if (WaitBlock->WaitKey != STATUS_TIMEOUT)
-        {
-            /* Wake the Object */
-            KiSatisfyObjectWait((PKMUTANT)WaitBlock->Object, WaitThread);
-        }
-
-        /* Move to the next block */
-        WaitBlock = WaitBlock->NextWaitBlock;
-    } while (WaitBlock != FirstBlock);
-}
-
 VOID
 FASTCALL
 KiWaitTest(IN PVOID ObjectPointer,
            IN KPRIORITY Increment)
 {
-    PLIST_ENTRY WaitEntry;
-    PLIST_ENTRY WaitList;
-    PKWAIT_BLOCK CurrentWaitBlock;
-    PKWAIT_BLOCK NextWaitBlock;
+    PLIST_ENTRY WaitEntry, WaitList;
+    PKWAIT_BLOCK WaitBlock;
     PKTHREAD WaitThread;
-    PKMUTANT FirstObject = ObjectPointer, Object;
+    PKMUTANT FirstObject = ObjectPointer;
+    NTSTATUS WaitStatus;
 
     /* Loop the Wait Entries */
     WaitList = &FirstObject->Header.WaitListHead;
     WaitEntry = WaitList->Flink;
-    while ((FirstObject->Header.SignalState > 0) &&
-           (WaitEntry != WaitList))
+    while ((FirstObject->Header.SignalState > 0) && (WaitEntry != WaitList))
     {
         /* Get the current wait block */
-        CurrentWaitBlock = CONTAINING_RECORD(WaitEntry,
-                                             KWAIT_BLOCK,
-                                             WaitListEntry);
-        WaitThread = CurrentWaitBlock->Thread;
+        WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
+        WaitThread = WaitBlock->Thread;
+        WaitStatus = STATUS_KERNEL_APC;
 
         /* Check the current Wait Mode */
-        if (CurrentWaitBlock->WaitType == WaitAny)
+        if (WaitBlock->WaitType == WaitAny)
         {
             /* Easy case, satisfy only this wait */
-            WaitEntry = WaitEntry->Blink;
+            WaitStatus = (NTSTATUS)WaitBlock->WaitKey;
             KiSatisfyObjectWait(FirstObject, WaitThread);
         }
-        else
-        {
-            /* Everything must be satisfied */
-            NextWaitBlock = CurrentWaitBlock->NextWaitBlock;
-
-            /* Loop first to make sure they are valid */
-            while (NextWaitBlock != CurrentWaitBlock)
-            {
-                /* Make sure this isn't a timeout block */
-                if (NextWaitBlock->WaitKey != STATUS_TIMEOUT)
-                {
-                    /* Get the object */
-                    Object = NextWaitBlock->Object;
-
-                    /* Check if this is a mutant */
-                    if ((Object->Header.Type == MutantObject) &&
-                        (Object->Header.SignalState <= 0) &&
-                        (WaitThread == Object->OwnerThread))
-                    {
-                        /* It's a signaled mutant */
-                    }
-                    else if (Object->Header.SignalState <= 0)
-                    {
-                        /* Skip the unwaiting */
-                        goto SkipUnwait;
-                    }
-                }
-
-                /* Go to the next Wait block */
-                NextWaitBlock = NextWaitBlock->NextWaitBlock;
-            }
-
-            /* All the objects are signaled, we can satisfy */
-            WaitEntry = WaitEntry->Blink;
-            KiWaitSatisfyAll(CurrentWaitBlock);
-        }
-
-        /* All waits satisfied, unwait the thread */
-        KiUnwaitThread(WaitThread, CurrentWaitBlock->WaitKey, Increment);
 
-SkipUnwait:
-        /* Next entry */
-        WaitEntry = WaitEntry->Flink;
+        /* Now do the rest of the unwait */
+        KiUnwaitThread(WaitThread, WaitStatus, Increment);
+        WaitEntry = WaitList->Flink;
     }
 }
 
@@ -141,13 +77,7 @@ KiUnlinkThread(IN PKTHREAD Thread,
 
     /* Check if there's a Thread Timer */
     Timer = &Thread->Timer;
-    if (Timer->Header.Inserted)
-    {
-        /* Remove the timer */
-        Timer->Header.Inserted = FALSE;
-        RemoveEntryList(&Timer->TimerListEntry);
-        //KiRemoveTimer(Timer);
-    }
+    if (Timer->Header.Inserted) KxRemoveTreeTimer(Timer);
 
     /* Increment the Queue's active threads */
     if (Thread->Queue) Thread->Queue->CurrentCount++;
@@ -157,7 +87,7 @@ KiUnlinkThread(IN PKTHREAD Thread,
 VOID
 FASTCALL
 KiUnwaitThread(IN PKTHREAD Thread,
-               IN NTSTATUS WaitStatus,
+               IN LONG_PTR WaitStatus,
                IN KPRIORITY Increment)
 {
     /* Unlink the thread */
@@ -166,7 +96,7 @@ KiUnwaitThread(IN PKTHREAD Thread,
     /* Tell the scheduler do to the increment when it readies the thread */
     ASSERT(Increment >= 0);
     Thread->AdjustIncrement = (SCHAR)Increment;
-    Thread->AdjustReason = 1;
+    Thread->AdjustReason = AdjustUnwait;
 
     /* Reschedule the Thread */
     KiReadyThread(Thread);
@@ -180,13 +110,83 @@ KiAcquireFastMutex(IN PFAST_MUTEX FastMutex)
     FastMutex->Contention++;
 
     /* Wait for the event */
-    KeWaitForSingleObject(&FastMutex->Gate,
+    KeWaitForSingleObject(&FastMutex->Event,
                           WrMutex,
                           KernelMode,
                           FALSE,
                           NULL);
 }
 
+VOID
+FASTCALL
+KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    ULONG BitsToRemove, BitsToAdd;
+    LONG OldValue, NewValue;
+
+    /* We depend on these bits being just right */
+    C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC);
+    
+    /* Increase the contention count */
+    GuardedMutex->Contention++;
+    
+    /* Start by unlocking the Guarded Mutex */
+    BitsToRemove = GM_LOCK_BIT;
+    BitsToAdd = GM_LOCK_WAITER_INC;
+    
+    /* Start change loop */
+    for (;;)
+    {
+        /* Loop sanity checks */
+        ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+               (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN)));
+        ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) ||
+               (BitsToAdd == GM_LOCK_WAITER_WOKEN));
+        
+        /* Get the Count Bits */
+        OldValue = GuardedMutex->Count;
+        
+        /* Start internal bit change loop */
+        for (;;)
+        {
+            /* Check if the Guarded Mutex is locked */
+            if (OldValue & GM_LOCK_BIT)
+            {
+                /* Sanity check */
+                ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+                       ((OldValue & GM_LOCK_WAITER_WOKEN) != 0));
+                
+                /* Unlock it by removing the Lock Bit */
+                NewValue = OldValue ^ BitsToRemove;
+                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+                                                      NewValue,
+                                                      OldValue);
+                if (NewValue == OldValue) return;
+            }
+            else
+            {
+                /* The Guarded Mutex isn't locked, so simply set the bits */
+                NewValue = OldValue + BitsToAdd;
+                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+                                                      NewValue,
+                                                      OldValue);
+                if (NewValue == OldValue) break;
+            }
+            
+            /* Old value changed, loop again */
+            OldValue = NewValue;
+        }
+        
+        /* Now we have to wait for it */
+        KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode);
+        ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0);
+        
+        /* Ok, the wait is done, so set the new bits */
+        BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN;
+        BitsToAdd = GM_LOCK_WAITER_WOKEN;
+    }
+}
+
 //
 // This routine exits the dispatcher after a compatible operation and
 // swaps the context to the next scheduled thread on the current CPU if
@@ -203,7 +203,7 @@ KiExitDispatcher(IN KIRQL OldIrql)
     BOOLEAN PendingApc;
 
     /* Make sure we're at synchronization level */
-    ASSERT_IRQL(SYNCH_LEVEL);
+    ASSERT(KeGetCurrentIrql() == SYNCH_LEVEL);
 
     /* Check if we have deferred threads */
     KiCheckDeferredReadyList(Prcb);
@@ -225,10 +225,6 @@ KiExitDispatcher(IN KIRQL OldIrql)
     /* Make sure there's a new thread scheduled */
     if (!Prcb->NextThread) goto Quickie;
 
-    /* This shouldn't happen on ROS yet */
-    DPRINT1("The impossible happened - Tell Alex\n");
-    ASSERT(FALSE);
-
     /* Lock the PRCB */
     KiAcquirePrcbLock(Prcb);
 
@@ -280,89 +276,123 @@ KeDelayExecutionThread(IN KPROCESSOR_MODE WaitMode,
                        IN BOOLEAN Alertable,
                        IN PLARGE_INTEGER Interval OPTIONAL)
 {
-    PKTIMER ThreadTimer;
+    PKTIMER Timer;
+    PKWAIT_BLOCK TimerBlock;
     PKTHREAD Thread = KeGetCurrentThread();
     NTSTATUS WaitStatus;
     BOOLEAN Swappable;
-    PLARGE_INTEGER OriginalDueTime = Interval;
-    LARGE_INTEGER DueTime, NewDueTime;
+    PLARGE_INTEGER OriginalDueTime;
+    LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
+    ULONG Hand = 0;
 
-    /* Check if the lock is already held */
-    if (Thread->WaitNext)
-    {
-        /* Lock is held, disable Wait Next */
-        Thread->WaitNext = FALSE;
-        Swappable = KxDelayThreadWait(Thread, Alertable, WaitMode);
-    }
-    else
+    /* If this is a user-mode wait of 0 seconds, yield execution */
+    if (!(Interval->QuadPart) && (WaitMode != KernelMode))
     {
-        /* Lock not held, acquire it */
-WaitStart:
-        Thread->WaitIrql = KiAcquireDispatcherLock();
-        Swappable = KxDelayThreadWait(Thread, Alertable, WaitMode);
+        /* Make sure the wait isn't alertable or interrupting an APC */
+        if (!(Alertable) && !(Thread->ApcState.UserApcPending))
+        {
+            /* Yield execution */
+            NtYieldExecution();
+        }
     }
 
-    /* Check if a kernel APC is pending and we're below APC_LEVEL */
-    if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
-        (Thread->WaitIrql < APC_LEVEL))
-    {
-        /* Unlock the dispatcher */
-        KiReleaseDispatcherLock(Thread->WaitIrql);
-        goto WaitStart;
-    }
+    /* Setup the original time and timer/wait blocks */
+    OriginalDueTime = Interval;
+    Timer = &Thread->Timer;
+    TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
 
-    /* Check if we have to bail out due to an alerted state */
-    WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
-    if (WaitStatus != STATUS_WAIT_0)
-    {
-        /* Unlock the dispatcher and return */
-        KiReleaseDispatcherLock(Thread->WaitIrql);
-        return WaitStatus;
-    }
+    /* Check if the lock is already held */
+    if (!Thread->WaitNext) goto WaitStart;
 
-    /* Set Timer */
-    ThreadTimer = &Thread->Timer;
+    /*  Otherwise, we already have the lock, so initialize the wait */
+    Thread->WaitNext = FALSE;
+    KxDelayThreadWait();
 
-    /* Insert the Timer into the Timer Lists and enable it */
-    if (!KiInsertTimer(ThreadTimer, *Interval))
+    /* Start wait loop */
+    for (;;)
     {
-        /* FIXME: We should find a new ready thread */
-        KiReleaseDispatcherLock(Thread->WaitIrql);
-        return STATUS_WAIT_0;
-    }
+        /* Disable pre-emption */
+        Thread->Preempted = FALSE;
 
-    /* Save due time */
-    DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
+        /* Check if a kernel APC is pending and we're below APC_LEVEL */
+        if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+            (Thread->WaitIrql < APC_LEVEL))
+        {
+            /* Unlock the dispatcher */
+            KiReleaseDispatcherLock(Thread->WaitIrql);
+        }
+        else
+        {
+            /* Check if we have to bail out due to an alerted state */
+            WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+            if (WaitStatus != STATUS_WAIT_0) break;
 
-    /* Handle Kernel Queues */
-    if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
+            /* Check if the timer expired */
+            InterruptTime.QuadPart = KeQueryInterruptTime();
+            if ((ULONGLONG)InterruptTime.QuadPart >= Timer->DueTime.QuadPart)
+            {
+                /* It did, so we don't need to wait */
+                goto NoWait;
+            }
 
-    /* Setup the wait information */
-    Thread->State = Waiting;
+            /* It didn't, so activate it */
+            Timer->Header.Inserted = TRUE;
 
-    /* Add the thread to the wait list */
-    KiAddThreadToWaitList(Thread, Swappable);
+            /* Handle Kernel Queues */
+            if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
 
-    /* Swap the thread */
-    ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
-    KiSetThreadSwapBusy(Thread);
-    WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
+            /* Setup the wait information */
+            Thread->State = Waiting;
 
-    /* Check if we were executing an APC or if we timed out */
-    if (WaitStatus == STATUS_KERNEL_APC)
-    {
-        /* Recalculate due times */
-        Interval = KiRecalculateDueTime(OriginalDueTime,
-                                        &DueTime,
-                                        &NewDueTime);
-        goto WaitStart;
-    }
+            /* Add the thread to the wait list */
+            KiAddThreadToWaitList(Thread, Swappable);
+
+            /* Insert the timer and swap the thread */
+            ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+            KiSetThreadSwapBusy(Thread);
+            KxInsertTimer(Timer, Hand);
+            WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
+
+            /* Check if were swapped ok */
+            if (WaitStatus != STATUS_KERNEL_APC)
+            {
+                /* This is a good thing */
+                if (WaitStatus == STATUS_TIMEOUT) WaitStatus = STATUS_SUCCESS;
 
-    /* This is a good thing */
-    if (WaitStatus == STATUS_TIMEOUT) WaitStatus = STATUS_SUCCESS;
+                /* Return Status */
+                return WaitStatus;
+            }
+
+            /* Recalculate due times */
+            Interval = KiRecalculateDueTime(OriginalDueTime,
+                                            &DueTime,
+                                            &NewDueTime);
+        }
+
+WaitStart:
+        /* Setup a new wait */
+        Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+        KxDelayThreadWait();
+        KiAcquireDispatcherLockAtDpcLevel();
+    }
 
-    /* Return Status */
+    /* We're done! */
+    KiReleaseDispatcherLock(Thread->WaitIrql);
     return WaitStatus;
+
+NoWait:
+    /* There was nothing to wait for. Did we have a wait interval? */
+    if (!Interval->QuadPart)
+    {
+        /* Unlock the dispatcher and do a yield */
+        KiReleaseDispatcherLock(Thread->WaitIrql);
+        return NtYieldExecution();
+    }
+
+    /* Unlock the dispatcher and adjust the quantum for a no-wait */
+    KiReleaseDispatcherLockFromDpcLevel();
+    KiAdjustQuantumThread(Thread);
+    return STATUS_SUCCESS;
 }
 
 /*
@@ -376,165 +406,155 @@ KeWaitForSingleObject(IN PVOID Object,
                       IN BOOLEAN Alertable,
                       IN PLARGE_INTEGER Timeout OPTIONAL)
 {
-    PKMUTANT CurrentObject;
-    PKWAIT_BLOCK WaitBlock;
-    PKTIMER ThreadTimer;
     PKTHREAD Thread = KeGetCurrentThread();
+    PKMUTANT CurrentObject = (PKMUTANT)Object;
+    PKWAIT_BLOCK WaitBlock = &Thread->WaitBlock[0];
+    PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+    PKTIMER Timer = &Thread->Timer;
     NTSTATUS WaitStatus;
     BOOLEAN Swappable;
-    LARGE_INTEGER DueTime, NewDueTime;
+    LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
     PLARGE_INTEGER OriginalDueTime = Timeout;
-
-    /* Get wait block */
-    WaitBlock = &Thread->WaitBlock[0];
+    ULONG Hand = 0;
 
     /* Check if the lock is already held */
-    if (Thread->WaitNext)
-    {
-        /* Lock is held, disable Wait Next */
-        Thread->WaitNext = FALSE;
-        Swappable = KxSingleThreadWait(Thread,
-                                       WaitBlock,
-                                       Object,
-                                       Timeout,
-                                       Alertable,
-                                       WaitReason,
-                                       WaitMode);
-    }
-    else
-    {
-StartWait:
-        /* Lock not held, acquire it */
-        Thread->WaitIrql = KiAcquireDispatcherLock();
-        Swappable = KxSingleThreadWait(Thread,
-                                       WaitBlock,
-                                       Object,
-                                       Timeout,
-                                       WaitReason,
-                                       WaitMode,
-                                       Alertable);
-    }
+    if (!Thread->WaitNext) goto WaitStart;
 
-    /* Check if a kernel APC is pending and we're below APC_LEVEL */
-    if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
-        (Thread->WaitIrql < APC_LEVEL))
-    {
-        /* Unlock the dispatcher and wait again */
-        KiReleaseDispatcherLock(Thread->WaitIrql);
-        goto StartWait;
-    }
-
-    /* Get the Current Object */
-    CurrentObject = (PKMUTANT)Object;
-    ASSERT(CurrentObject->Header.Type != QueueObject);
+    /*  Otherwise, we already have the lock, so initialize the wait */
+    Thread->WaitNext = FALSE;
+    KxSingleThreadWait();
 
-    /* Check if it's a mutant */
-    if (CurrentObject->Header.Type == MutantObject)
+    /* Start wait loop */
+    for (;;)
     {
-        /* Check its signal state or if we own it */
-        if ((CurrentObject->Header.SignalState > 0) ||
-            (Thread == CurrentObject->OwnerThread))
+        /* Disable pre-emption */
+        Thread->Preempted = FALSE;
+
+        /* Check if a kernel APC is pending and we're below APC_LEVEL */
+        if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+            (Thread->WaitIrql < APC_LEVEL))
+        {
+            /* Unlock the dispatcher */
+            KiReleaseDispatcherLock(Thread->WaitIrql);
+        }
+        else
         {
-            /* Just unwait this guy and exit */
-            if (CurrentObject->Header.SignalState != (LONG)MINLONG)
+            /* Sanity check */
+            ASSERT(CurrentObject->Header.Type != QueueObject);
+
+            /* Check if it's a mutant */
+            if (CurrentObject->Header.Type == MutantObject)
+            {
+                /* Check its signal state or if we own it */
+                if ((CurrentObject->Header.SignalState > 0) ||
+                    (Thread == CurrentObject->OwnerThread))
+                {
+                    /* Just unwait this guy and exit */
+                    if (CurrentObject->Header.SignalState != (LONG)MINLONG)
+                    {
+                        /* It has a normal signal state. Unwait and return */
+                        KiSatisfyMutantWait(CurrentObject, Thread);
+                        WaitStatus = Thread->WaitStatus;
+                        goto DontWait;
+                    }
+                    else
+                    {
+                        /* Raise an exception */
+                        KiReleaseDispatcherLock(Thread->WaitIrql);
+                        ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
+                   }
+                }
+            }
+            else if (CurrentObject->Header.SignalState > 0)
             {
-                /* It has a normal signal state. Unwait and return */
-                KiSatisfyMutantWait(CurrentObject, Thread);
-                WaitStatus = Thread->WaitStatus;
+                /* Another satisfied object */
+                KiSatisfyNonMutantWait(CurrentObject);
+                WaitStatus = STATUS_WAIT_0;
                 goto DontWait;
             }
-            else
+
+            /* Make sure we can satisfy the Alertable request */
+            WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+            if (WaitStatus != STATUS_WAIT_0) break;
+
+            /* Enable the Timeout Timer if there was any specified */
+            if (Timeout)
             {
-                /* Raise an exception */
-                KiReleaseDispatcherLock(Thread->WaitIrql);
-                ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
-           }
-        }
-    }
-    else if (CurrentObject->Header.SignalState > 0)
-    {
-        /* Another satisfied object */
-        KiSatisfyNonMutantWait(CurrentObject, Thread);
-        WaitStatus = STATUS_WAIT_0;
-        goto DontWait;
-    }
+                /* Check if the timer expired */
+                InterruptTime.QuadPart = KeQueryInterruptTime();
+                if ((ULONGLONG)InterruptTime.QuadPart >=
+                    Timer->DueTime.QuadPart)
+                {
+                    /* It did, so we don't need to wait */
+                    WaitStatus = STATUS_TIMEOUT;
+                    goto DontWait;
+                }
 
-    /* Make sure we can satisfy the Alertable request */
-    WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
-    if (WaitStatus != STATUS_WAIT_0)
-    {
-        /* Unlock the dispatcher and return */
-        KiReleaseDispatcherLock(Thread->WaitIrql);
-        return WaitStatus;
-    }
+                /* It didn't, so activate it */
+                Timer->Header.Inserted = TRUE;
+            }
 
-    /* Enable the Timeout Timer if there was any specified */
-    if (Timeout)
-    {
-        /* Fail if the timeout interval is actually 0 */
-        if (!Timeout->QuadPart)
-        {
-            /* Return a timeout */
-            WaitStatus = STATUS_TIMEOUT;
-            goto DontWait;
-        }
+            /* Link the Object to this Wait Block */
+            InsertTailList(&CurrentObject->Header.WaitListHead,
+                           &WaitBlock->WaitListEntry);
 
-        /* Insert the Timer into the Timer Lists and enable it */
-        ThreadTimer = &Thread->Timer;
-        if (!KiInsertTimer(ThreadTimer, *Timeout))
-        {
-            /* Return a timeout if we couldn't insert the timer */
-            WaitStatus = STATUS_TIMEOUT;
-            goto DontWait;
-        }
+            /* Handle Kernel Queues */
+            if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
 
-        /* Set the current due time */
-        DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
-    }
+            /* Setup the wait information */
+            Thread->State = Waiting;
 
-    /* Link the Object to this Wait Block */
-    InsertTailList(&CurrentObject->Header.WaitListHead,
-                   &WaitBlock->WaitListEntry);
+            /* Add the thread to the wait list */
+            KiAddThreadToWaitList(Thread, Swappable);
 
-    /* Handle Kernel Queues */
-    if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
+            /* Activate thread swap */
+            ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+            KiSetThreadSwapBusy(Thread);
 
-    /* Setup the wait information */
-    Thread->State = Waiting;
+            /* Check if we have a timer */
+            if (Timeout)
+            {
+                /* Insert it */
+                KxInsertTimer(Timer, Hand);
+            }
+            else
+            {
+                /* Otherwise, unlock the dispatcher */
+                KiReleaseDispatcherLockFromDpcLevel();
+            }
 
-    /* Add the thread to the wait list */
-    KiAddThreadToWaitList(Thread, Swappable);
+            /* Do the actual swap */
+            WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
 
-    /* Swap the thread */
-    ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
-    KiSetThreadSwapBusy(Thread);
-    WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
+            /* Check if we were executing an APC */
+            if (WaitStatus != STATUS_KERNEL_APC) return WaitStatus;
 
-    /* Check if we were executing an APC */
-    if (WaitStatus == STATUS_KERNEL_APC)
-    {
-        /* Check if we had a timeout */
-        if (Timeout)
-        {
-            /* Recalculate due times */
-            Timeout = KiRecalculateDueTime(OriginalDueTime,
-                                           &DueTime,
-                                           &NewDueTime);
+            /* Check if we had a timeout */
+            if (Timeout)
+            {
+                /* Recalculate due times */
+                Timeout = KiRecalculateDueTime(OriginalDueTime,
+                                               &DueTime,
+                                               &NewDueTime);
+            }
         }
-
-        /* Wait again */
-        goto StartWait;
+WaitStart:
+        /* Setup a new wait */
+        Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+        KxSingleThreadWait();
+        KiAcquireDispatcherLockAtDpcLevel();
     }
 
     /* Wait complete */
+    KiReleaseDispatcherLock(Thread->WaitIrql);
     return WaitStatus;
 
 DontWait:
-    /* Adjust the Quantum */
-    KiAdjustQuantumThread(Thread);
+    /* Release dispatcher lock but maintain high IRQL */
+    KiReleaseDispatcherLockFromDpcLevel();
 
-    /* Release & Return */
-    KiReleaseDispatcherLock(Thread->WaitIrql);
+    /* Adjust the Quantum and return the wait status */
+    KiAdjustQuantumThread(Thread);
     return WaitStatus;
 }
 
@@ -554,30 +574,14 @@ KeWaitForMultipleObjects(IN ULONG Count,
 {
     PKMUTANT CurrentObject;
     PKWAIT_BLOCK WaitBlock;
-    PKWAIT_BLOCK TimerWaitBlock;
-    PKTIMER ThreadTimer;
-    PKTHREAD CurrentThread = KeGetCurrentThread();
-    ULONG AllObjectsSignaled;
-    ULONG WaitIndex;
+    PKTHREAD Thread = KeGetCurrentThread();
+    PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+    PKTIMER Timer = &Thread->Timer;
     NTSTATUS WaitStatus = STATUS_SUCCESS;
     BOOLEAN Swappable;
     PLARGE_INTEGER OriginalDueTime = Timeout;
-    LARGE_INTEGER DueTime, NewDueTime;
-
-    /* Set the Current Thread */
-    CurrentThread = KeGetCurrentThread();
-
-    /* Check if the lock is already held */
-    if (CurrentThread->WaitNext)
-    {
-        /* Lock is held, disable Wait Next */
-        CurrentThread->WaitNext = FALSE;
-    }
-    else
-    {
-        /* Lock not held, acquire it */
-        CurrentThread->WaitIrql = KiAcquireDispatcherLock();
-    }
+    LARGE_INTEGER DueTime, NewDueTime, InterruptTime;
+    ULONG Index, Hand = 0;
 
     /* Make sure the Wait Count is valid */
     if (!WaitBlockArray)
@@ -586,11 +590,11 @@ KeWaitForMultipleObjects(IN ULONG Count,
         if (Count > THREAD_WAIT_OBJECTS)
         {
             /* Bugcheck */
-            KEBUGCHECK(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
+            KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
         }
 
         /* Use the Thread's Wait Block */
-        WaitBlockArray = &CurrentThread->WaitBlock[0];
+        WaitBlockArray = &Thread->WaitBlock[0];
     }
     else
     {
@@ -598,64 +602,72 @@ KeWaitForMultipleObjects(IN ULONG Count,
         if (Count > MAXIMUM_WAIT_OBJECTS)
         {
             /* Bugcheck */
-            KEBUGCHECK(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
+            KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
         }
     }
 
-    /* Start the actual Loop */
-    do
+    /* Sanity check */
+    ASSERT(Count != 0);
+
+    /* Check if the lock is already held */
+    if (!Thread->WaitNext) goto WaitStart;
+
+    /*  Otherwise, we already have the lock, so initialize the wait */
+    Thread->WaitNext = FALSE;
+    /*  Note that KxMultiThreadWait is a macro, defined in ke_x.h, that  */
+    /*  uses  (and modifies some of) the following local                 */
+    /*  variables:                                                       */
+    /*  Thread, Index, WaitBlock, Timer, Timeout, Hand and Swappable.    */
+    /*  If it looks like this code doesn't actually wait for any objects */
+    /*  at all, it's because the setup is done by that macro.            */
+    KxMultiThreadWait();
+
+    /* Start wait loop */
+    for (;;)
     {
+        /* Disable pre-emption */
+        Thread->Preempted = FALSE;
+
         /* Check if a kernel APC is pending and we're below APC_LEVEL */
-        if ((CurrentThread->ApcState.KernelApcPending) &&
-            !(CurrentThread->SpecialApcDisable) &&
-            (CurrentThread->WaitIrql < APC_LEVEL))
+        if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+            (Thread->WaitIrql < APC_LEVEL))
         {
             /* Unlock the dispatcher */
-            KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+            KiReleaseDispatcherLock(Thread->WaitIrql);
         }
         else
         {
-            /* Append wait block to the KTHREAD wait block list */
-            CurrentThread->WaitBlockList = WaitBlock = WaitBlockArray;
-
-            /* Set default wait status */
-            CurrentThread->WaitStatus = STATUS_WAIT_0;
-
-            /* Check if the wait is (already) satisfied */
-            AllObjectsSignaled = TRUE;
-
-            /* First, we'll try to satisfy the wait directly */
-            for (WaitIndex = 0; WaitIndex < Count; WaitIndex++)
+            /* Check what kind of wait this is */
+            Index = 0;
+            if (WaitType == WaitAny)
             {
-                /* Get the Current Object */
-                CurrentObject = (PKMUTANT)Object[WaitIndex];
-
-                /* Check the type of wait */
-                if (WaitType == WaitAny)
+                /* Loop blocks */
+                do
                 {
+                    /* Get the Current Object */
+                    CurrentObject = (PKMUTANT)Object[Index];
+                    ASSERT(CurrentObject->Header.Type != QueueObject);
+
                     /* Check if the Object is a mutant */
                     if (CurrentObject->Header.Type == MutantObject)
                     {
                         /* Check if it's signaled */
                         if ((CurrentObject->Header.SignalState > 0) ||
-                            (CurrentThread == CurrentObject->OwnerThread))
+                            (Thread == CurrentObject->OwnerThread))
                         {
                             /* This is a Wait Any, so unwait this and exit */
                             if (CurrentObject->Header.SignalState !=
                                 (LONG)MINLONG)
                             {
                                 /* Normal signal state, unwait it and return */
-                                KiSatisfyMutantWait(CurrentObject,
-                                                    CurrentThread);
-                                WaitStatus = CurrentThread->WaitStatus |
-                                             WaitIndex;
+                                KiSatisfyMutantWait(CurrentObject, Thread);
+                                WaitStatus = Thread->WaitStatus | Index;
                                 goto DontWait;
                             }
                             else
                             {
                                 /* Raise an exception (see wasm.ru) */
-                                KiReleaseDispatcherLock(CurrentThread->
-                                                                WaitIrql);
+                                KiReleaseDispatcherLock(Thread->WaitIrql);
                                 ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
                             }
                         }
@@ -663,111 +675,99 @@ KeWaitForMultipleObjects(IN ULONG Count,
                     else if (CurrentObject->Header.SignalState > 0)
                     {
                         /* Another signaled object, unwait and return */
-                        KiSatisfyNonMutantWait(CurrentObject, CurrentThread);
-                        WaitStatus = WaitIndex;
+                        KiSatisfyNonMutantWait(CurrentObject);
+                        WaitStatus = Index;
                         goto DontWait;
                     }
-                }
-                else
+
+                    /* Go to the next block */
+                    Index++;
+                } while (Index < Count);
+            }
+            else
+            {
+                /* Loop blocks */
+                do
                 {
+                    /* Get the Current Object */
+                    CurrentObject = (PKMUTANT)Object[Index];
+                    ASSERT(CurrentObject->Header.Type != QueueObject);
+
                     /* Check if we're dealing with a mutant again */
                     if (CurrentObject->Header.Type == MutantObject)
                     {
                         /* Check if it has an invalid count */
-                        if ((CurrentThread == CurrentObject->OwnerThread) &&
-                            (CurrentObject->Header.SignalState == MINLONG))
+                        if ((Thread == CurrentObject->OwnerThread) &&
+                            (CurrentObject->Header.SignalState == (LONG)MINLONG))
                         {
                             /* Raise an exception */
-                            KiReleaseDispatcherLock(CurrentThread->
-                                                            WaitIrql);
+                            KiReleaseDispatcherLock(Thread->WaitIrql);
                             ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
                         }
                         else if ((CurrentObject->Header.SignalState <= 0) &&
-                                 (CurrentThread != CurrentObject->OwnerThread))
+                                 (Thread != CurrentObject->OwnerThread))
                         {
                             /* We don't own it, can't satisfy the wait */
-                            AllObjectsSignaled = FALSE;
+                            break;
                         }
                     }
                     else if (CurrentObject->Header.SignalState <= 0)
                     {
                         /* Not signaled, can't satisfy */
-                        AllObjectsSignaled = FALSE;
+                        break;
                     }
-                }
 
-                /* Set up a Wait Block for this Object */
-                WaitBlock->Object = CurrentObject;
-                WaitBlock->Thread = CurrentThread;
-                WaitBlock->WaitKey = (USHORT)WaitIndex;
-                WaitBlock->WaitType = (UCHAR)WaitType;
-                WaitBlock->NextWaitBlock = WaitBlock + 1;
+                    /* Go to the next block */
+                    Index++;
+                } while (Index < Count);
 
-                /* Move to the next Wait Block */
-                WaitBlock = WaitBlock->NextWaitBlock;
-            }
-
-            /* Return to the Root Wait Block */
-            WaitBlock--;
-            WaitBlock->NextWaitBlock = WaitBlockArray;
+                /* Check if we've went through all the objects */
+                if (Index == Count)
+                {
+                    /* Loop wait blocks */
+                    WaitBlock = WaitBlockArray;
+                    do
+                    {
+                        /* Get the object and satisfy it */
+                        CurrentObject = (PKMUTANT)WaitBlock->Object;
+                        KiSatisfyObjectWait(CurrentObject, Thread);
 
-            /* Check if this is a Wait All and all the objects are signaled */
-            if ((WaitType == WaitAll) && (AllObjectsSignaled))
-            {
-                /* Return to the Root Wait Block */
-                WaitBlock = CurrentThread->WaitBlockList;
+                        /* Go to the next block */
+                        WaitBlock = WaitBlock->NextWaitBlock;
+                    } while(WaitBlock != WaitBlockArray);
 
-                /* Satisfy their Waits and return to the caller */
-                KiWaitSatisfyAll(WaitBlock);
-                WaitStatus = CurrentThread->WaitStatus;
-                goto DontWait;
+                    /* Set the wait status and get out */
+                    WaitStatus = Thread->WaitStatus;
+                    goto DontWait;
+                }
             }
 
             /* Make sure we can satisfy the Alertable request */
-            WaitStatus = KiCheckAlertability(CurrentThread, Alertable, WaitMode);
-
-            /* Check if we can swap the thread's stack */
-            CurrentThread->WaitListEntry.Flink = NULL;
-            Swappable = KiCheckThreadStackSwap(CurrentThread, WaitMode);
+            WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
+            if (WaitStatus != STATUS_WAIT_0) break;
 
             /* Enable the Timeout Timer if there was any specified */
             if (Timeout)
             {
-                /* Make sure the timeout interval isn't actually 0 */
-                if (!Timeout->QuadPart)
+                /* Check if the timer expired */
+                InterruptTime.QuadPart = KeQueryInterruptTime();
+                if ((ULONGLONG)InterruptTime.QuadPart >=
+                    Timer->DueTime.QuadPart)
                 {
-                    /* Return a timeout */
+                    /* It did, so we don't need to wait */
                     WaitStatus = STATUS_TIMEOUT;
                     goto DontWait;
                 }
 
-                /* Point to Timer Wait Block and Thread Timer */
-                TimerWaitBlock = &CurrentThread->WaitBlock[TIMER_WAIT_BLOCK];
-                ThreadTimer = &CurrentThread->Timer;
-
-                /* Connect the Timer Wait Block */
-                WaitBlock->NextWaitBlock = TimerWaitBlock;
-
-                /* Set up the Timer Wait Block */
-                TimerWaitBlock->NextWaitBlock = WaitBlockArray;
+                /* It didn't, so activate it */
+                Timer->Header.Inserted = TRUE;
 
-                /* Initialize the list head */
-                InitializeListHead(&ThreadTimer->Header.WaitListHead);
-
-                /* Insert the Timer into the Timer Lists and enable it */
-                if (!KiInsertTimer(ThreadTimer, *Timeout))
-                {
-                    /* Return a timeout if we couldn't insert the timer */
-                    WaitStatus = STATUS_TIMEOUT;
-                    goto DontWait;
-                }
-
-                /* Set the current due time */
-                DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
+                /* Link the wait blocks */
+                WaitBlock->NextWaitBlock = TimerBlock;
             }
 
             /* Insert into Object's Wait List*/
-            WaitBlock = CurrentThread->WaitBlockList;
+            WaitBlock = WaitBlockArray;
             do
             {
                 /* Get the Current Object */
@@ -782,19 +782,32 @@ KeWaitForMultipleObjects(IN ULONG Count,
             } while (WaitBlock != WaitBlockArray);
 
             /* Handle Kernel Queues */
-            if (CurrentThread->Queue) KiActivateWaiterQueue(CurrentThread->Queue);
+            if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
 
             /* Setup the wait information */
-            CurrentThread->Alertable = Alertable;
-            CurrentThread->WaitMode = WaitMode;
-            CurrentThread->WaitReason = WaitReason;
-            CurrentThread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
-            CurrentThread->State = Waiting;
+            Thread->State = Waiting;
+
+            /* Add the thread to the wait list */
+            KiAddThreadToWaitList(Thread, Swappable);
 
-            /* Find a new thread to run */
-            KiAddThreadToWaitList(CurrentThread, Swappable);
-            WaitStatus = KiSwapThread(CurrentThread, KeGetCurrentPrcb());
-            ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
+            /* Activate thread swap */
+            ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+            KiSetThreadSwapBusy(Thread);
+
+            /* Check if we have a timer */
+            if (Timeout)
+            {
+                /* Insert it */
+                KxInsertTimer(Timer, Hand);
+            }
+            else
+            {
+                /* Otherwise, unlock the dispatcher */
+                KiReleaseDispatcherLockFromDpcLevel();
+            }
+
+            /* Swap the thread */
+            WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
 
             /* Check if we were executing an APC */
             if (WaitStatus != STATUS_KERNEL_APC) return WaitStatus;
@@ -807,22 +820,25 @@ KeWaitForMultipleObjects(IN ULONG Count,
                                                &DueTime,
                                                &NewDueTime);
             }
-
-            /* Acquire again the lock */
-            CurrentThread->WaitIrql = KiAcquireDispatcherLock();
         }
-    } while (TRUE);
 
-    /* Release the Lock, we are done */
-    KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+WaitStart:
+        /* Setup a new wait */
+        Thread->WaitIrql = KeRaiseIrqlToSynchLevel();
+        KxMultiThreadWait();
+        KiAcquireDispatcherLockAtDpcLevel();
+    }
+
+    /* We are done */
+    KiReleaseDispatcherLock(Thread->WaitIrql);
     return WaitStatus;
 
 DontWait:
-    /* Adjust the Quantum */
-    KiAdjustQuantumThread(CurrentThread);
+    /* Release dispatcher lock but maintain high IRQL */
+    KiReleaseDispatcherLockFromDpcLevel();
 
-    /* Release & Return */
-    KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+    /* Adjust the Quantum and return the wait status */
+    KiAdjustQuantumThread(Thread);
     return WaitStatus;
 }
 
@@ -833,25 +849,24 @@ NtDelayExecution(IN BOOLEAN Alertable,
 {
     KPROCESSOR_MODE PreviousMode = ExGetPreviousMode();
     LARGE_INTEGER SafeInterval;
-    NTSTATUS Status = STATUS_SUCCESS;
+    NTSTATUS Status;
 
     /* Check the previous mode */
-    if(PreviousMode != KernelMode)
+    if (PreviousMode != KernelMode)
     {
         /* Enter SEH for probing */
-        _SEH_TRY
+        _SEH2_TRY
         {
             /* Probe and capture the time out */
             SafeInterval = ProbeForReadLargeInteger(DelayInterval);
             DelayInterval = &SafeInterval;
         }
-        _SEH_HANDLE
+        _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
         {
-            /* Get SEH exception */
-            Status = _SEH_GetExceptionCode();
+            /* Return the exception code */
+            _SEH2_YIELD(return _SEH2_GetExceptionCode());
         }
-        _SEH_END;
-        if (!NT_SUCCESS(Status)) return Status;
+        _SEH2_END;
    }
 
    /* Call the Kernel Function */