2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
12 KeGetPreviousMode(VOID
)
14 /* Return the current mode */
15 return KeGetCurrentThread()->PreviousMode
;
20 // Enters a Guarded Region
22 #define KeEnterGuardedRegion() \
24 PKTHREAD _Thread = KeGetCurrentThread(); \
27 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
28 ASSERT(_Thread == KeGetCurrentThread()); \
29 ASSERT((_Thread->SpecialApcDisable <= 0) && \
30 (_Thread->SpecialApcDisable != -32768)); \
32 /* Disable Special APCs */ \
33 _Thread->SpecialApcDisable--; \
37 // Leaves a Guarded Region
39 #define KeLeaveGuardedRegion() \
41 PKTHREAD _Thread = KeGetCurrentThread(); \
44 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
45 ASSERT(_Thread == KeGetCurrentThread()); \
46 ASSERT(_Thread->SpecialApcDisable < 0); \
48 /* Leave region and check if APCs are OK now */ \
49 if (!(++_Thread->SpecialApcDisable)) \
51 /* Check for Kernel APCs on the list */ \
52 if (!IsListEmpty(&_Thread->ApcState. \
53 ApcListHead[KernelMode])) \
55 /* Check for APC Delivery */ \
56 KiCheckForKernelApcDelivery(); \
62 // Enters a Critical Region
64 #define KeEnterCriticalRegion() \
66 PKTHREAD _Thread = KeGetCurrentThread(); \
69 ASSERT(_Thread == KeGetCurrentThread()); \
70 ASSERT((_Thread->KernelApcDisable <= 0) && \
71 (_Thread->KernelApcDisable != -32768)); \
73 /* Disable Kernel APCs */ \
74 _Thread->KernelApcDisable--; \
78 // Leaves a Critical Region
80 #define KeLeaveCriticalRegion() \
82 PKTHREAD _Thread = KeGetCurrentThread(); \
85 ASSERT(_Thread == KeGetCurrentThread()); \
86 ASSERT(_Thread->KernelApcDisable < 0); \
88 /* Enable Kernel APCs */ \
89 _Thread->KernelApcDisable++; \
91 /* Check if Kernel APCs are now enabled */ \
92 if (!(_Thread->KernelApcDisable)) \
94 /* Check if we need to request an APC Delivery */ \
95 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
96 !(_Thread->SpecialApcDisable)) \
98 /* Check for the right environment */ \
99 KiCheckForKernelApcDelivery(); \
106 Kii386SpinOnSpinLock(PKSPIN_LOCK SpinLock
, ULONG Flags
);
110 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
114 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
116 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
117 UNREFERENCED_PARAMETER(SpinLock
);
121 // Spinlock Release at IRQL >= DISPATCH_LEVEL
125 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
127 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
128 UNREFERENCED_PARAMETER(SpinLock
);
132 // This routine protects against multiple CPU acquires, it's meaningless on UP.
136 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
138 UNREFERENCED_PARAMETER(Object
);
142 // This routine protects against multiple CPU acquires, it's meaningless on UP.
146 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
148 UNREFERENCED_PARAMETER(Object
);
153 KiAcquireDispatcherLock(VOID
)
155 /* Raise to DPC level */
156 return KeRaiseIrqlToDpcLevel();
161 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
163 /* Just exit the dispatcher */
164 KiExitDispatcher(OldIrql
);
169 KiAcquireDispatcherLockAtDpcLevel(VOID
)
171 /* This is a no-op at DPC Level for UP systems */
177 KiReleaseDispatcherLockFromDpcLevel(VOID
)
179 /* This is a no-op at DPC Level for UP systems */
184 // This routine makes the thread deferred ready on the boot CPU.
188 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
190 /* Set the thread to deferred state and boot CPU */
191 Thread
->State
= DeferredReady
;
192 Thread
->DeferredProcessor
= 0;
194 /* Make the thread ready immediately */
195 KiDeferredReadyThread(Thread
);
200 KiRescheduleThread(IN BOOLEAN NewThread
,
203 /* This is meaningless on UP systems */
204 UNREFERENCED_PARAMETER(NewThread
);
205 UNREFERENCED_PARAMETER(Cpu
);
209 // This routine protects against multiple CPU acquires, it's meaningless on UP.
213 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
215 UNREFERENCED_PARAMETER(Thread
);
219 // This routine protects against multiple CPU acquires, it's meaningless on UP.
223 KiAcquirePrcbLock(IN PKPRCB Prcb
)
225 UNREFERENCED_PARAMETER(Prcb
);
229 // This routine protects against multiple CPU acquires, it's meaningless on UP.
233 KiReleasePrcbLock(IN PKPRCB Prcb
)
235 UNREFERENCED_PARAMETER(Prcb
);
239 // This routine protects against multiple CPU acquires, it's meaningless on UP.
243 KiAcquireThreadLock(IN PKTHREAD Thread
)
245 UNREFERENCED_PARAMETER(Thread
);
249 // This routine protects against multiple CPU acquires, it's meaningless on UP.
253 KiReleaseThreadLock(IN PKTHREAD Thread
)
255 UNREFERENCED_PARAMETER(Thread
);
259 // This routine protects against multiple CPU acquires, it's meaningless on UP.
263 KiTryThreadLock(IN PKTHREAD Thread
)
265 UNREFERENCED_PARAMETER(Thread
);
271 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
273 /* There are no deferred ready lists on UP systems */
274 UNREFERENCED_PARAMETER(Prcb
);
279 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
282 /* We deliver instantly on UP */
283 UNREFERENCED_PARAMETER(NeedApc
);
284 UNREFERENCED_PARAMETER(Processor
);
289 KiAcquireTimerLock(IN ULONG Hand
)
291 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
293 /* Nothing to do on UP */
294 UNREFERENCED_PARAMETER(Hand
);
300 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
302 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
304 /* Nothing to do on UP */
305 UNREFERENCED_PARAMETER(LockQueue
);
311 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
315 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
318 /* Make sure that we don't own the lock already */
319 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) == *SpinLock
)
321 /* We do, bugcheck! */
322 KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
326 /* Try to acquire the lock */
327 while (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
329 #if defined(_M_IX86) && defined(DBG)
330 /* On x86 debug builds, we use a much slower but useful routine */
331 Kii386SpinOnSpinLock(SpinLock
, 5);
333 /* It's locked... spin until it's unlocked */
334 while (*(volatile KSPIN_LOCK
*)SpinLock
& 1)
336 /* Yield and keep looping */
342 /* On debug builds, we OR in the KTHREAD */
343 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
348 // Spinlock Release at IRQL >= DISPATCH_LEVEL
352 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
355 /* Make sure that the threads match */
356 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
358 /* They don't, bugcheck */
359 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
363 InterlockedAnd((PLONG
)SpinLock
, 0);
368 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
372 /* Make sure we're at a safe level to touch the lock */
373 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
375 /* Start acquire loop */
378 /* Loop until the other CPU releases it */
381 /* Check if it got released */
382 OldValue
= Object
->Lock
;
383 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
385 /* Let the CPU know that this is a loop */
389 /* Try acquiring the lock now */
390 } while (InterlockedCompareExchange(&Object
->Lock
,
391 OldValue
| KOBJECT_LOCK_BIT
,
392 OldValue
) != OldValue
);
397 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
399 /* Make sure we're at a safe level to touch the lock */
400 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
403 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
408 KiAcquireDispatcherLock(VOID
)
410 /* Raise to synchronization level and acquire the dispatcher lock */
411 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
416 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
418 /* First release the lock */
419 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
420 LockQueue
[LockQueueDispatcherLock
]);
422 /* Then exit the dispatcher */
423 KiExitDispatcher(OldIrql
);
428 KiAcquireDispatcherLockAtDpcLevel(VOID
)
430 /* Acquire the dispatcher lock */
431 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
432 LockQueue
[LockQueueDispatcherLock
]);
437 KiReleaseDispatcherLockFromDpcLevel(VOID
)
439 /* Release the dispatcher lock */
440 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
441 LockQueue
[LockQueueDispatcherLock
]);
445 // This routine inserts a thread into the deferred ready list of the current CPU
449 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
451 PKPRCB Prcb
= KeGetCurrentPrcb();
453 /* Set the thread to deferred state and CPU */
454 Thread
->State
= DeferredReady
;
455 Thread
->DeferredProcessor
= Prcb
->Number
;
457 /* Add it on the list */
458 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
463 KiRescheduleThread(IN BOOLEAN NewThread
,
466 /* Check if a new thread needs to be scheduled on a different CPU */
467 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
469 /* Send an IPI to request delivery */
470 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
475 // This routine sets the current thread in a swap busy state, which ensure that
476 // nobody else tries to swap it concurrently.
480 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
482 /* Make sure nobody already set it */
483 ASSERT(Thread
->SwapBusy
== FALSE
);
485 /* Set it ourselves */
486 Thread
->SwapBusy
= TRUE
;
490 // This routine acquires the PRCB lock so that only one caller can touch
491 // volatile PRCB data.
493 // Since this is a simple optimized spin-lock, it must only be acquired
494 // at dispatcher level or higher!
498 KiAcquirePrcbLock(IN PKPRCB Prcb
)
500 /* Make sure we're at a safe level to touch the PRCB lock */
501 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
503 /* Start acquire loop */
506 /* Acquire the lock and break out if we acquired it first */
507 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
509 /* Loop until the other CPU releases it */
512 /* Let the CPU know that this is a loop */
514 } while (Prcb
->PrcbLock
);
519 // This routine releases the PRCB lock so that other callers can touch
520 // volatile PRCB data.
522 // Since this is a simple optimized spin-lock, it must be be only acquired
523 // at dispatcher level or higher!
527 KiReleasePrcbLock(IN PKPRCB Prcb
)
529 /* Make sure we are above dispatch and the lock is acquired! */
530 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
531 ASSERT(Prcb
->PrcbLock
!= 0);
534 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
538 // This routine acquires the thread lock so that only one caller can touch
539 // volatile thread data.
541 // Since this is a simple optimized spin-lock, it must be be only acquired
542 // at dispatcher level or higher!
546 KiAcquireThreadLock(IN PKTHREAD Thread
)
548 /* Make sure we're at a safe level to touch the thread lock */
549 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
551 /* Start acquire loop */
554 /* Acquire the lock and break out if we acquired it first */
555 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
557 /* Loop until the other CPU releases it */
560 /* Let the CPU know that this is a loop */
562 } while (Thread
->ThreadLock
);
567 // This routine releases the thread lock so that other callers can touch
568 // volatile thread data.
570 // Since this is a simple optimized spin-lock, it must be be only acquired
571 // at dispatcher level or higher!
575 KiReleaseThreadLock(IN PKTHREAD Thread
)
577 /* Make sure we are still above dispatch */
578 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
581 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
586 KiTryThreadLock(IN PKTHREAD Thread
)
590 /* If the lock isn't acquired, return false */
591 if (!Thread
->ThreadLock
) return FALSE
;
593 /* Otherwise, try to acquire it and check the result */
595 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
597 /* Return the lock state */
598 return (Value
== TRUE
);
603 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
605 /* Scan the deferred ready lists if required */
606 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
611 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
614 /* Check if we need to request APC delivery */
617 /* Check if it's on another CPU */
618 if (KeGetPcr()->Number
!= Processor
)
620 /* Send an IPI to request delivery */
621 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
625 /* Request a software interrupt */
626 HalRequestSoftwareInterrupt(APC_LEVEL
);
633 KiAcquireTimerLock(IN ULONG Hand
)
635 PKSPIN_LOCK_QUEUE LockQueue
;
637 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
639 /* Get the lock index */
640 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
641 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
643 /* Now get the lock */
644 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
646 /* Acquire it and return */
647 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
653 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
655 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
657 /* Release the lock */
658 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
665 KiAcquireApcLock(IN PKTHREAD Thread
,
666 IN PKLOCK_QUEUE_HANDLE Handle
)
668 /* Acquire the lock and raise to synchronization level */
669 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
674 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
675 IN PKLOCK_QUEUE_HANDLE Handle
)
677 /* Acquire the lock */
678 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
683 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
684 IN PKLOCK_QUEUE_HANDLE Handle
)
686 /* Acquire the lock */
687 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
692 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
694 /* Release the lock */
695 KeReleaseInStackQueuedSpinLock(Handle
);
700 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
702 /* Release the lock */
703 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
708 KiAcquireProcessLock(IN PKPROCESS Process
,
709 IN PKLOCK_QUEUE_HANDLE Handle
)
711 /* Acquire the lock and raise to synchronization level */
712 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
717 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
719 /* Release the lock */
720 KeReleaseInStackQueuedSpinLock(Handle
);
725 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
727 /* Release the lock */
728 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
733 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
734 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
736 /* Check if we were called from a threaded DPC */
737 if (KeGetCurrentPrcb()->DpcThreadActive
)
739 /* Lock the Queue, we're not at DPC level */
740 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
744 /* We must be at DPC level, acquire the lock safely */
745 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
746 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
753 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
755 /* Check if we were called from a threaded DPC */
756 if (KeGetCurrentPrcb()->DpcThreadActive
)
758 /* Unlock the Queue, we're not at DPC level */
759 KeReleaseInStackQueuedSpinLock(DeviceLock
);
763 /* We must be at DPC level, release the lock safely */
764 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
765 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
770 // Satisfies the wait of any dispatcher object
772 #define KiSatisfyObjectWait(Object, Thread) \
774 /* Special case for Mutants */ \
775 if ((Object)->Header.Type == MutantObject) \
777 /* Decrease the Signal State */ \
778 (Object)->Header.SignalState--; \
780 /* Check if it's now non-signaled */ \
781 if (!(Object)->Header.SignalState) \
783 /* Set the Owner Thread */ \
784 (Object)->OwnerThread = Thread; \
786 /* Disable APCs if needed */ \
787 Thread->KernelApcDisable = Thread->KernelApcDisable - \
788 (Object)->ApcDisable; \
790 /* Check if it's abandoned */ \
791 if ((Object)->Abandoned) \
794 (Object)->Abandoned = FALSE; \
796 /* Return Status */ \
797 Thread->WaitStatus = STATUS_ABANDONED; \
800 /* Insert it into the Mutant List */ \
801 InsertHeadList(Thread->MutantListHead.Blink, \
802 &(Object)->MutantListEntry); \
805 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
806 EventSynchronizationObject) \
808 /* Synchronization Timers and Events just get un-signaled */ \
809 (Object)->Header.SignalState = 0; \
811 else if ((Object)->Header.Type == SemaphoreObject) \
813 /* These ones can have multiple states, so we only decrease it */ \
814 (Object)->Header.SignalState--; \
819 // Satisfies the wait of a mutant dispatcher object
821 #define KiSatisfyMutantWait(Object, Thread) \
823 /* Decrease the Signal State */ \
824 (Object)->Header.SignalState--; \
826 /* Check if it's now non-signaled */ \
827 if (!(Object)->Header.SignalState) \
829 /* Set the Owner Thread */ \
830 (Object)->OwnerThread = Thread; \
832 /* Disable APCs if needed */ \
833 Thread->KernelApcDisable = Thread->KernelApcDisable - \
834 (Object)->ApcDisable; \
836 /* Check if it's abandoned */ \
837 if ((Object)->Abandoned) \
840 (Object)->Abandoned = FALSE; \
842 /* Return Status */ \
843 Thread->WaitStatus = STATUS_ABANDONED; \
846 /* Insert it into the Mutant List */ \
847 InsertHeadList(Thread->MutantListHead.Blink, \
848 &(Object)->MutantListEntry); \
853 // Satisfies the wait of any nonmutant dispatcher object
855 #define KiSatisfyNonMutantWait(Object) \
857 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
858 EventSynchronizationObject) \
860 /* Synchronization Timers and Events just get un-signaled */ \
861 (Object)->Header.SignalState = 0; \
863 else if ((Object)->Header.Type == SemaphoreObject) \
865 /* These ones can have multiple states, so we only decrease it */ \
866 (Object)->Header.SignalState--; \
871 // Recalculates the due time
875 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
876 IN PLARGE_INTEGER DueTime
,
877 IN OUT PLARGE_INTEGER NewDueTime
)
879 /* Don't do anything for absolute waits */
880 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
882 /* Otherwise, query the interrupt time and recalculate */
883 NewDueTime
->QuadPart
= KeQueryInterruptTime();
884 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
889 // Determines whether a thread should be added to the wait list
893 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
894 IN KPROCESSOR_MODE WaitMode
)
896 /* Check the required conditions */
897 if ((WaitMode
!= KernelMode
) &&
898 (Thread
->EnableStackSwap
) &&
899 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
901 /* We are go for swap */
906 /* Don't swap the thread */
912 // Adds a thread to the wait list
914 #define KiAddThreadToWaitList(Thread, Swappable) \
916 /* Make sure it's swappable */ \
919 /* Insert it into the PRCB's List */ \
920 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
921 &Thread->WaitListEntry); \
926 // Checks if a wait in progress should be interrupted by APCs or an alertable
931 KiCheckAlertability(IN PKTHREAD Thread
,
932 IN BOOLEAN Alertable
,
933 IN KPROCESSOR_MODE WaitMode
)
935 /* Check if the wait is alertable */
938 /* It is, first check if the thread is alerted in this mode */
939 if (Thread
->Alerted
[WaitMode
])
941 /* It is, so bail out of the wait */
942 Thread
->Alerted
[WaitMode
] = FALSE
;
943 return STATUS_ALERTED
;
945 else if ((WaitMode
!= KernelMode
) &&
946 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
948 /* It's isn't, but this is a user wait with queued user APCs */
949 Thread
->ApcState
.UserApcPending
= TRUE
;
950 return STATUS_USER_APC
;
952 else if (Thread
->Alerted
[KernelMode
])
954 /* It isn't that either, but we're alered in kernel mode */
955 Thread
->Alerted
[KernelMode
] = FALSE
;
956 return STATUS_ALERTED
;
959 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
961 /* Not alertable, but this is a user wait with pending user APCs */
962 return STATUS_USER_APC
;
965 /* Otherwise, we're fine */
966 return STATUS_WAIT_0
;
970 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
971 // to remove timer entries
972 // See Windows HPI blog for more information.
975 KiRemoveEntryTimer(IN PKTIMER Timer
)
978 PKTIMER_TABLE_ENTRY TableEntry
;
980 /* Remove the timer from the timer list and check if it's empty */
981 Hand
= Timer
->Header
.Hand
;
982 if (RemoveEntryList(&Timer
->TimerListEntry
))
984 /* Get the respective timer table entry */
985 TableEntry
= &KiTimerTableListHead
[Hand
];
986 if (&TableEntry
->Entry
== TableEntry
->Entry
.Flink
)
988 /* Set the entry to an infinite absolute time */
989 TableEntry
->Time
.HighPart
= 0xFFFFFFFF;
993 /* Clear the list entries on dbg builds so we can tell the timer is gone */
995 Timer
->TimerListEntry
.Flink
= NULL
;
996 Timer
->TimerListEntry
.Blink
= NULL
;
1001 // Called by Wait and Queue code to insert a timer for dispatching.
1002 // Also called by KeSetTimerEx to insert a timer from the caller.
1006 KxInsertTimer(IN PKTIMER Timer
,
1009 PKSPIN_LOCK_QUEUE LockQueue
;
1011 /* Acquire the lock and release the dispatcher lock */
1012 LockQueue
= KiAcquireTimerLock(Hand
);
1013 KiReleaseDispatcherLockFromDpcLevel();
1015 /* Try to insert the timer */
1016 if (KiInsertTimerTable(Timer
, Hand
))
1019 KiCompleteTimer(Timer
, LockQueue
);
1023 /* Do nothing, just release the lock */
1024 KiReleaseTimerLock(LockQueue
);
1029 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
1030 // See the Windows HPI Blog for more information
1034 KiComputeDueTime(IN PKTIMER Timer
,
1035 IN LARGE_INTEGER DueTime
,
1038 LARGE_INTEGER InterruptTime
, SystemTime
, DifferenceTime
;
1040 /* Convert to relative time if needed */
1041 Timer
->Header
.Absolute
= FALSE
;
1042 if (DueTime
.HighPart
>= 0)
1044 /* Get System Time */
1045 KeQuerySystemTime(&SystemTime
);
1047 /* Do the conversion */
1048 DifferenceTime
.QuadPart
= SystemTime
.QuadPart
- DueTime
.QuadPart
;
1050 /* Make sure it hasn't already expired */
1051 Timer
->Header
.Absolute
= TRUE
;
1052 if (DifferenceTime
.HighPart
>= 0)
1054 /* Cancel everything */
1055 Timer
->Header
.SignalState
= TRUE
;
1056 Timer
->Header
.Hand
= 0;
1057 Timer
->DueTime
.QuadPart
= 0;
1062 /* Set the time as Absolute */
1063 DueTime
= DifferenceTime
;
1066 /* Get the Interrupt Time */
1067 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1069 /* Recalculate due time */
1070 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- DueTime
.QuadPart
;
1072 /* Get the handle */
1073 *Hand
= KiComputeTimerTableIndex(Timer
->DueTime
.QuadPart
);
1074 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1075 Timer
->Header
.Inserted
= TRUE
;
1080 // Called from Unlink and Queue Insert Code.
1081 // Also called by timer code when canceling an inserted timer.
1082 // Removes a timer from it's tree.
1086 KxRemoveTreeTimer(IN PKTIMER Timer
)
1088 ULONG Hand
= Timer
->Header
.Hand
;
1089 PKSPIN_LOCK_QUEUE LockQueue
;
1090 PKTIMER_TABLE_ENTRY TimerEntry
;
1092 /* Acquire timer lock */
1093 LockQueue
= KiAcquireTimerLock(Hand
);
1095 /* Set the timer as non-inserted */
1096 Timer
->Header
.Inserted
= FALSE
;
1098 /* Remove it from the timer list */
1099 if (RemoveEntryList(&Timer
->TimerListEntry
))
1101 /* Get the entry and check if it's empty */
1102 TimerEntry
= &KiTimerTableListHead
[Hand
];
1103 if (IsListEmpty(&TimerEntry
->Entry
))
1105 /* Clear the time then */
1106 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1110 /* Release the timer lock */
1111 KiReleaseTimerLock(LockQueue
);
1116 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1117 IN LARGE_INTEGER Interval
,
1121 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1123 /* Check the timer's interval to see if it's absolute */
1124 Timer
->Header
.Absolute
= FALSE
;
1125 if (Interval
.HighPart
>= 0)
1127 /* Get the system time and calculate the relative time */
1128 KeQuerySystemTime(&SystemTime
);
1129 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1130 Timer
->Header
.Absolute
= TRUE
;
1132 /* Check if we've already expired */
1133 if (TimeDifference
.HighPart
>= 0)
1135 /* Reset everything */
1136 Timer
->DueTime
.QuadPart
= 0;
1138 Timer
->Header
.Hand
= 0;
1143 /* Update the interval */
1144 Interval
= TimeDifference
;
1148 /* Calculate the due time */
1149 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1150 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1151 Timer
->DueTime
.QuadPart
= DueTime
;
1153 /* Calculate the timer handle */
1154 *Hand
= KiComputeTimerTableIndex(DueTime
);
1155 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1158 #define KxDelayThreadWait() \
1160 /* Setup the Wait Block */ \
1161 Thread->WaitBlockList = TimerBlock; \
1163 /* Setup the timer */ \
1164 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1166 /* Save the due time for the caller */ \
1167 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1169 /* Link the timer to this Wait Block */ \
1170 TimerBlock->NextWaitBlock = TimerBlock; \
1171 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1172 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1174 /* Clear wait status */ \
1175 Thread->WaitStatus = STATUS_SUCCESS; \
1177 /* Setup wait fields */ \
1178 Thread->Alertable = Alertable; \
1179 Thread->WaitReason = DelayExecution; \
1180 Thread->WaitMode = WaitMode; \
1182 /* Check if we can swap the thread's stack */ \
1183 Thread->WaitListEntry.Flink = NULL; \
1184 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1186 /* Set the wait time */ \
1187 Thread->WaitTime = KeTickCount.LowPart;
1189 #define KxMultiThreadWait() \
1190 /* Link wait block array to the thread */ \
1191 Thread->WaitBlockList = WaitBlockArray; \
1193 /* Reset the index */ \
1196 /* Loop wait blocks */ \
1199 /* Fill out the wait block */ \
1200 WaitBlock = &WaitBlockArray[Index]; \
1201 WaitBlock->Object = Object[Index]; \
1202 WaitBlock->WaitKey = (USHORT)Index; \
1203 WaitBlock->WaitType = WaitType; \
1204 WaitBlock->Thread = Thread; \
1206 /* Link to next block */ \
1207 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1209 } while (Index < Count); \
1211 /* Link the last block */ \
1212 WaitBlock->NextWaitBlock = WaitBlockArray; \
1214 /* Set default wait status */ \
1215 Thread->WaitStatus = STATUS_WAIT_0; \
1217 /* Check if we have a timer */ \
1220 /* Link to the block */ \
1221 TimerBlock->NextWaitBlock = WaitBlockArray; \
1223 /* Setup the timer */ \
1224 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1226 /* Save the due time for the caller */ \
1227 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1229 /* Initialize the list */ \
1230 InitializeListHead(&Timer->Header.WaitListHead); \
1233 /* Set wait settings */ \
1234 Thread->Alertable = Alertable; \
1235 Thread->WaitMode = WaitMode; \
1236 Thread->WaitReason = WaitReason; \
1238 /* Check if we can swap the thread's stack */ \
1239 Thread->WaitListEntry.Flink = NULL; \
1240 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1242 /* Set the wait time */ \
1243 Thread->WaitTime = KeTickCount.LowPart;
1245 #define KxSingleThreadWait() \
1246 /* Setup the Wait Block */ \
1247 Thread->WaitBlockList = WaitBlock; \
1248 WaitBlock->WaitKey = STATUS_SUCCESS; \
1249 WaitBlock->Object = Object; \
1250 WaitBlock->WaitType = WaitAny; \
1252 /* Clear wait status */ \
1253 Thread->WaitStatus = STATUS_SUCCESS; \
1255 /* Check if we have a timer */ \
1258 /* Setup the timer */ \
1259 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1261 /* Save the due time for the caller */ \
1262 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1264 /* Pointer to timer block */ \
1265 WaitBlock->NextWaitBlock = TimerBlock; \
1266 TimerBlock->NextWaitBlock = WaitBlock; \
1268 /* Link the timer to this Wait Block */ \
1269 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1270 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1274 /* No timer block, just ourselves */ \
1275 WaitBlock->NextWaitBlock = WaitBlock; \
1278 /* Set wait settings */ \
1279 Thread->Alertable = Alertable; \
1280 Thread->WaitMode = WaitMode; \
1281 Thread->WaitReason = WaitReason; \
1283 /* Check if we can swap the thread's stack */ \
1284 Thread->WaitListEntry.Flink = NULL; \
1285 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1287 /* Set the wait time */ \
1288 Thread->WaitTime = KeTickCount.LowPart;
1290 #define KxQueueThreadWait() \
1291 /* Setup the Wait Block */ \
1292 Thread->WaitBlockList = WaitBlock; \
1293 WaitBlock->WaitKey = STATUS_SUCCESS; \
1294 WaitBlock->Object = Queue; \
1295 WaitBlock->WaitType = WaitAny; \
1296 WaitBlock->Thread = Thread; \
1298 /* Clear wait status */ \
1299 Thread->WaitStatus = STATUS_SUCCESS; \
1301 /* Check if we have a timer */ \
1304 /* Setup the timer */ \
1305 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1307 /* Save the due time for the caller */ \
1308 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1310 /* Pointer to timer block */ \
1311 WaitBlock->NextWaitBlock = TimerBlock; \
1312 TimerBlock->NextWaitBlock = WaitBlock; \
1314 /* Link the timer to this Wait Block */ \
1315 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1316 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1320 /* No timer block, just ourselves */ \
1321 WaitBlock->NextWaitBlock = WaitBlock; \
1324 /* Set wait settings */ \
1325 Thread->Alertable = FALSE; \
1326 Thread->WaitMode = WaitMode; \
1327 Thread->WaitReason = WrQueue; \
1329 /* Check if we can swap the thread's stack */ \
1330 Thread->WaitListEntry.Flink = NULL; \
1331 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1333 /* Set the wait time */ \
1334 Thread->WaitTime = KeTickCount.LowPart;
1341 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1342 IN KPRIORITY Increment
)
1344 PLIST_ENTRY WaitEntry
, WaitList
;
1345 PKWAIT_BLOCK WaitBlock
;
1346 PKTHREAD WaitThread
;
1349 /* Loop the Wait Entries */
1350 WaitList
= &Object
->WaitListHead
;
1351 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1352 WaitEntry
= WaitList
->Flink
;
1355 /* Get the current wait block */
1356 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1358 /* Get the waiting thread */
1359 WaitThread
= WaitBlock
->Thread
;
1361 /* Check the current Wait Mode */
1362 if (WaitBlock
->WaitType
== WaitAny
)
1364 /* Use the actual wait key */
1365 WaitKey
= WaitBlock
->WaitKey
;
1369 /* Otherwise, use STATUS_KERNEL_APC */
1370 WaitKey
= STATUS_KERNEL_APC
;
1373 /* Unwait the thread */
1374 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1377 WaitEntry
= WaitList
->Flink
;
1378 } while (WaitEntry
!= WaitList
);
1382 // Unwaits a Thread waiting on an event
1386 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1387 IN KPRIORITY Increment
)
1389 PLIST_ENTRY WaitEntry
, WaitList
;
1390 PKWAIT_BLOCK WaitBlock
;
1391 PKTHREAD WaitThread
;
1393 /* Loop the Wait Entries */
1394 WaitList
= &Event
->Header
.WaitListHead
;
1395 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1396 WaitEntry
= WaitList
->Flink
;
1399 /* Get the current wait block */
1400 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1402 /* Get the waiting thread */
1403 WaitThread
= WaitBlock
->Thread
;
1405 /* Check the current Wait Mode */
1406 if (WaitBlock
->WaitType
== WaitAny
)
1409 Event
->Header
.SignalState
= 0;
1411 /* Un-signal the event and unwait the thread */
1412 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1416 /* Unwait the thread with STATUS_KERNEL_APC */
1417 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1420 WaitEntry
= WaitList
->Flink
;
1421 } while (WaitEntry
!= WaitList
);
1425 // This routine queues a thread that is ready on the PRCB's ready lists.
1426 // If this thread cannot currently run on this CPU, then the thread is
1427 // added to the deferred ready list instead.
1429 // This routine must be entered with the PRCB lock held and it will exit
1430 // with the PRCB lock released!
1434 KxQueueReadyThread(IN PKTHREAD Thread
,
1441 ASSERT(Prcb
== KeGetCurrentPrcb());
1442 ASSERT(Thread
->State
== Running
);
1443 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1445 /* Check if this thread is allowed to run in this CPU */
1447 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1452 /* Set thread ready for execution */
1453 Thread
->State
= Ready
;
1455 /* Save current priority and if someone had pre-empted it */
1456 Priority
= Thread
->Priority
;
1457 Preempted
= Thread
->Preempted
;
1459 /* We're not pre-empting now, and set the wait time */
1460 Thread
->Preempted
= FALSE
;
1461 Thread
->WaitTime
= KeTickCount
.LowPart
;
1464 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1466 /* Insert this thread in the appropriate order */
1467 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1468 &Thread
->WaitListEntry
) :
1469 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1470 &Thread
->WaitListEntry
);
1472 /* Update the ready summary */
1473 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1476 ASSERT(Priority
== Thread
->Priority
);
1478 /* Release the PRCB lock */
1479 KiReleasePrcbLock(Prcb
);
1483 /* Otherwise, prepare this thread to be deferred */
1484 Thread
->State
= DeferredReady
;
1485 Thread
->DeferredProcessor
= Prcb
->Number
;
1487 /* Release the lock and defer scheduling */
1488 KiReleasePrcbLock(Prcb
);
1489 KiDeferredReadyThread(Thread
);
1494 // This routine scans for an appropriate ready thread to select at the
1495 // given priority and for the given CPU.
1499 KiSelectReadyThread(IN KPRIORITY Priority
,
1504 PLIST_ENTRY ListEntry
;
1505 PKTHREAD Thread
= NULL
;
1507 /* Save the current mask and get the priority set for the CPU */
1508 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1509 if (!PrioritySet
) goto Quickie
;
1511 /* Get the highest priority possible */
1512 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1513 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1514 HighPriority
+= Priority
;
1516 /* Make sure the list isn't empty at the highest priority */
1517 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1519 /* Get the first thread on the list */
1520 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1521 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1523 /* Make sure this thread is here for a reason */
1524 ASSERT(HighPriority
== Thread
->Priority
);
1525 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1526 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1528 /* Remove it from the list */
1529 if (RemoveEntryList(&Thread
->WaitListEntry
))
1531 /* The list is empty now, reset the ready summary */
1532 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1535 /* Sanity check and return the thread */
1537 ASSERT((Thread
== NULL
) ||
1538 (Thread
->BasePriority
== 0) ||
1539 (Thread
->Priority
!= 0));
1544 // This routine computes the new priority for a thread. It is only valid for
1545 // threads with priorities in the dynamic priority range.
1549 KiComputeNewPriority(IN PKTHREAD Thread
,
1550 IN SCHAR Adjustment
)
1554 /* Priority sanity checks */
1555 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1556 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1557 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1558 TRUE
: (Thread
->PriorityDecrement
== 0));
1560 /* Get the current priority */
1561 Priority
= Thread
->Priority
;
1562 if (Priority
< LOW_REALTIME_PRIORITY
)
1564 /* Decrease priority by the priority decrement */
1565 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1567 /* Don't go out of bounds */
1568 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1570 /* Reset the priority decrement */
1571 Thread
->PriorityDecrement
= 0;
1575 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1577 /* Return the new priority */
1582 // Guarded Mutex Routines
1586 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex
)
1588 /* Setup the Initial Data */
1589 GuardedMutex
->Count
= GM_LOCK_BIT
;
1590 GuardedMutex
->Owner
= NULL
;
1591 GuardedMutex
->Contention
= 0;
1593 /* Initialize the Wait Gate */
1594 KeInitializeGate(&GuardedMutex
->Gate
);
1599 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1601 PKTHREAD Thread
= KeGetCurrentThread();
1604 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1605 (Thread
->SpecialApcDisable
< 0) ||
1606 (Thread
->Teb
== NULL
) ||
1607 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1608 ASSERT(GuardedMutex
->Owner
!= Thread
);
1610 /* Remove the lock */
1611 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1613 /* The Guarded Mutex was already locked, enter contented case */
1614 KiAcquireGuardedMutex(GuardedMutex
);
1618 GuardedMutex
->Owner
= Thread
;
1623 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1625 LONG OldValue
, NewValue
;
1628 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1629 (KeGetCurrentThread()->SpecialApcDisable
< 0) ||
1630 (KeGetCurrentThread()->Teb
== NULL
) ||
1631 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1632 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1634 /* Destroy the Owner */
1635 GuardedMutex
->Owner
= NULL
;
1637 /* Add the Lock Bit */
1638 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1639 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1641 /* Check if it was already locked, but not woken */
1642 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1644 /* Update the Oldvalue to what it should be now */
1645 OldValue
+= GM_LOCK_BIT
;
1647 /* The mutex will be woken, minus one waiter */
1648 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1651 /* Remove the Woken bit */
1652 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1654 OldValue
) == OldValue
)
1656 /* Signal the Gate */
1657 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1664 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex
)
1666 PKTHREAD Thread
= KeGetCurrentThread();
1669 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1670 ASSERT(GuardedMutex
->Owner
!= Thread
);
1672 /* Disable Special APCs */
1673 KeEnterGuardedRegion();
1675 /* Remove the lock */
1676 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1678 /* The Guarded Mutex was already locked, enter contented case */
1679 KiAcquireGuardedMutex(GuardedMutex
);
1682 /* Set the Owner and Special APC Disable state */
1683 GuardedMutex
->Owner
= Thread
;
1684 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;
1689 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1691 LONG OldValue
, NewValue
;
1694 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1695 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1696 ASSERT(KeGetCurrentThread()->SpecialApcDisable
==
1697 GuardedMutex
->SpecialApcDisable
);
1699 /* Destroy the Owner */
1700 GuardedMutex
->Owner
= NULL
;
1702 /* Add the Lock Bit */
1703 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1704 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1706 /* Check if it was already locked, but not woken */
1707 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1709 /* Update the Oldvalue to what it should be now */
1710 OldValue
+= GM_LOCK_BIT
;
1712 /* The mutex will be woken, minus one waiter */
1713 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1716 /* Remove the Woken bit */
1717 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1719 OldValue
) == OldValue
)
1721 /* Signal the Gate */
1722 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1726 /* Re-enable APCs */
1727 KeLeaveGuardedRegion();
1732 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1734 PKTHREAD Thread
= KeGetCurrentThread();
1737 KeEnterGuardedRegion();
1739 /* Remove the lock */
1740 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1742 /* Re-enable APCs */
1743 KeLeaveGuardedRegion();
1746 /* Return failure */
1750 /* Set the Owner and APC State */
1751 GuardedMutex
->Owner
= Thread
;
1752 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;