2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
12 KeGetPreviousMode(VOID
)
14 /* Return the current mode */
15 return KeGetCurrentThread()->PreviousMode
;
20 // Enters a Guarded Region
22 #define KeEnterGuardedRegion() \
24 PKTHREAD _Thread = KeGetCurrentThread(); \
27 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
28 ASSERT(_Thread == KeGetCurrentThread()); \
29 ASSERT((_Thread->SpecialApcDisable <= 0) && \
30 (_Thread->SpecialApcDisable != -32768)); \
32 /* Disable Special APCs */ \
33 _Thread->SpecialApcDisable--; \
37 // Leaves a Guarded Region
39 #define KeLeaveGuardedRegion() \
41 PKTHREAD _Thread = KeGetCurrentThread(); \
44 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
45 ASSERT(_Thread == KeGetCurrentThread()); \
46 ASSERT(_Thread->SpecialApcDisable < 0); \
48 /* Leave region and check if APCs are OK now */ \
49 if (!(++_Thread->SpecialApcDisable)) \
51 /* Check for Kernel APCs on the list */ \
52 if (!IsListEmpty(&_Thread->ApcState. \
53 ApcListHead[KernelMode])) \
55 /* Check for APC Delivery */ \
56 KiCheckForKernelApcDelivery(); \
62 // Enters a Critical Region
64 #define KeEnterCriticalRegion() \
66 PKTHREAD _Thread = KeGetCurrentThread(); \
69 ASSERT(_Thread == KeGetCurrentThread()); \
70 ASSERT((_Thread->KernelApcDisable <= 0) && \
71 (_Thread->KernelApcDisable != -32768)); \
73 /* Disable Kernel APCs */ \
74 _Thread->KernelApcDisable--; \
78 // Leaves a Critical Region
80 #define KeLeaveCriticalRegion() \
82 PKTHREAD _Thread = KeGetCurrentThread(); \
85 ASSERT(_Thread == KeGetCurrentThread()); \
86 ASSERT(_Thread->KernelApcDisable < 0); \
88 /* Enable Kernel APCs */ \
89 _Thread->KernelApcDisable++; \
91 /* Check if Kernel APCs are now enabled */ \
92 if (!(_Thread->KernelApcDisable)) \
94 /* Check if we need to request an APC Delivery */ \
95 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
96 !(_Thread->SpecialApcDisable)) \
98 /* Check for the right environment */ \
99 KiCheckForKernelApcDelivery(); \
106 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
110 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
112 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
113 UNREFERENCED_PARAMETER(SpinLock
);
117 // Spinlock Release at IRQL >= DISPATCH_LEVEL
121 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
123 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
124 UNREFERENCED_PARAMETER(SpinLock
);
128 // This routine protects against multiple CPU acquires, it's meaningless on UP.
132 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
134 UNREFERENCED_PARAMETER(Object
);
138 // This routine protects against multiple CPU acquires, it's meaningless on UP.
142 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
144 UNREFERENCED_PARAMETER(Object
);
149 KiAcquireDispatcherLock(VOID
)
151 /* Raise to DPC level */
152 return KeRaiseIrqlToDpcLevel();
157 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
159 /* Just exit the dispatcher */
160 KiExitDispatcher(OldIrql
);
165 KiAcquireDispatcherLockAtDpcLevel(VOID
)
167 /* This is a no-op at DPC Level for UP systems */
173 KiReleaseDispatcherLockFromDpcLevel(VOID
)
175 /* This is a no-op at DPC Level for UP systems */
180 // This routine makes the thread deferred ready on the boot CPU.
184 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
186 /* Set the thread to deferred state and boot CPU */
187 Thread
->State
= DeferredReady
;
188 Thread
->DeferredProcessor
= 0;
190 /* Make the thread ready immediately */
191 KiDeferredReadyThread(Thread
);
196 KiRescheduleThread(IN BOOLEAN NewThread
,
199 /* This is meaningless on UP systems */
200 UNREFERENCED_PARAMETER(NewThread
);
201 UNREFERENCED_PARAMETER(Cpu
);
205 // This routine protects against multiple CPU acquires, it's meaningless on UP.
209 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
211 UNREFERENCED_PARAMETER(Thread
);
215 // This routine protects against multiple CPU acquires, it's meaningless on UP.
219 KiAcquirePrcbLock(IN PKPRCB Prcb
)
221 UNREFERENCED_PARAMETER(Prcb
);
225 // This routine protects against multiple CPU acquires, it's meaningless on UP.
229 KiReleasePrcbLock(IN PKPRCB Prcb
)
231 UNREFERENCED_PARAMETER(Prcb
);
235 // This routine protects against multiple CPU acquires, it's meaningless on UP.
239 KiAcquireThreadLock(IN PKTHREAD Thread
)
241 UNREFERENCED_PARAMETER(Thread
);
245 // This routine protects against multiple CPU acquires, it's meaningless on UP.
249 KiReleaseThreadLock(IN PKTHREAD Thread
)
251 UNREFERENCED_PARAMETER(Thread
);
255 // This routine protects against multiple CPU acquires, it's meaningless on UP.
259 KiTryThreadLock(IN PKTHREAD Thread
)
261 UNREFERENCED_PARAMETER(Thread
);
267 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
269 /* There are no deferred ready lists on UP systems */
270 UNREFERENCED_PARAMETER(Prcb
);
275 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
278 /* We deliver instantly on UP */
279 UNREFERENCED_PARAMETER(NeedApc
);
280 UNREFERENCED_PARAMETER(Processor
);
285 KiAcquireTimerLock(IN ULONG Hand
)
287 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
289 /* Nothing to do on UP */
290 UNREFERENCED_PARAMETER(Hand
);
296 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
298 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
300 /* Nothing to do on UP */
301 UNREFERENCED_PARAMETER(LockQueue
);
307 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
311 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
313 /* Make sure that we don't own the lock already */
314 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) == *SpinLock
)
316 /* We do, bugcheck! */
317 KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
320 /* Start acquire loop */
323 /* Try to acquire it */
324 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
326 /* Value changed... wait until it's unlocked */
327 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
330 /* On debug builds, we use a much slower but useful routine */
331 //Kii386SpinOnSpinLock(SpinLock, 5);
333 /* FIXME: Do normal yield for now */
336 /* Otherwise, just yield and keep looping */
344 /* On debug builds, we OR in the KTHREAD */
345 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
347 /* All is well, break out */
354 // Spinlock Release at IRQL >= DISPATCH_LEVEL
358 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
361 /* Make sure that the threads match */
362 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
364 /* They don't, bugcheck */
365 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
369 InterlockedAnd((PLONG
)SpinLock
, 0);
374 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
378 /* Make sure we're at a safe level to touch the lock */
379 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
381 /* Start acquire loop */
384 /* Loop until the other CPU releases it */
387 /* Check if it got released */
388 OldValue
= Object
->Lock
;
389 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
391 /* Let the CPU know that this is a loop */
395 /* Try acquiring the lock now */
396 } while (InterlockedCompareExchange(&Object
->Lock
,
397 OldValue
| KOBJECT_LOCK_BIT
,
398 OldValue
) != OldValue
);
403 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
405 /* Make sure we're at a safe level to touch the lock */
406 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
409 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
414 KiAcquireDispatcherLock(VOID
)
416 /* Raise to synchronization level and acquire the dispatcher lock */
417 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
422 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
424 /* First release the lock */
425 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
426 LockQueue
[LockQueueDispatcherLock
]);
428 /* Then exit the dispatcher */
429 KiExitDispatcher(OldIrql
);
434 KiAcquireDispatcherLockAtDpcLevel(VOID
)
436 /* Acquire the dispatcher lock */
437 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
438 LockQueue
[LockQueueDispatcherLock
]);
443 KiReleaseDispatcherLockFromDpcLevel(VOID
)
445 /* Release the dispatcher lock */
446 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
447 LockQueue
[LockQueueDispatcherLock
]);
451 // This routine inserts a thread into the deferred ready list of the current CPU
455 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
457 PKPRCB Prcb
= KeGetCurrentPrcb();
459 /* Set the thread to deferred state and CPU */
460 Thread
->State
= DeferredReady
;
461 Thread
->DeferredProcessor
= Prcb
->Number
;
463 /* Add it on the list */
464 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
469 KiRescheduleThread(IN BOOLEAN NewThread
,
472 /* Check if a new thread needs to be scheduled on a different CPU */
473 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
475 /* Send an IPI to request delivery */
476 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
481 // This routine sets the current thread in a swap busy state, which ensure that
482 // nobody else tries to swap it concurrently.
486 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
488 /* Make sure nobody already set it */
489 ASSERT(Thread
->SwapBusy
== FALSE
);
491 /* Set it ourselves */
492 Thread
->SwapBusy
= TRUE
;
496 // This routine acquires the PRCB lock so that only one caller can touch
497 // volatile PRCB data.
499 // Since this is a simple optimized spin-lock, it must only be acquired
500 // at dispatcher level or higher!
504 KiAcquirePrcbLock(IN PKPRCB Prcb
)
506 /* Make sure we're at a safe level to touch the PRCB lock */
507 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
509 /* Start acquire loop */
512 /* Acquire the lock and break out if we acquired it first */
513 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
515 /* Loop until the other CPU releases it */
518 /* Let the CPU know that this is a loop */
520 } while (Prcb
->PrcbLock
);
525 // This routine releases the PRCB lock so that other callers can touch
526 // volatile PRCB data.
528 // Since this is a simple optimized spin-lock, it must be be only acquired
529 // at dispatcher level or higher!
533 KiReleasePrcbLock(IN PKPRCB Prcb
)
535 /* Make sure we are above dispatch and the lock is acquired! */
536 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
537 ASSERT(Prcb
->PrcbLock
!= 0);
540 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
544 // This routine acquires the thread lock so that only one caller can touch
545 // volatile thread data.
547 // Since this is a simple optimized spin-lock, it must be be only acquired
548 // at dispatcher level or higher!
552 KiAcquireThreadLock(IN PKTHREAD Thread
)
554 /* Make sure we're at a safe level to touch the thread lock */
555 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
557 /* Start acquire loop */
560 /* Acquire the lock and break out if we acquired it first */
561 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
563 /* Loop until the other CPU releases it */
566 /* Let the CPU know that this is a loop */
568 } while (Thread
->ThreadLock
);
573 // This routine releases the thread lock so that other callers can touch
574 // volatile thread data.
576 // Since this is a simple optimized spin-lock, it must be be only acquired
577 // at dispatcher level or higher!
581 KiReleaseThreadLock(IN PKTHREAD Thread
)
583 /* Make sure we are still above dispatch */
584 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
587 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
592 KiTryThreadLock(IN PKTHREAD Thread
)
596 /* If the lock isn't acquired, return false */
597 if (!Thread
->ThreadLock
) return FALSE
;
599 /* Otherwise, try to acquire it and check the result */
601 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
603 /* Return the lock state */
604 return (Value
== TRUE
);
609 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
611 /* Scan the deferred ready lists if required */
612 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
617 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
620 /* Check if we need to request APC delivery */
623 /* Check if it's on another CPU */
624 if (KeGetPcr()->Number
!= Processor
)
626 /* Send an IPI to request delivery */
627 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
631 /* Request a software interrupt */
632 HalRequestSoftwareInterrupt(APC_LEVEL
);
639 KiAcquireTimerLock(IN ULONG Hand
)
641 PKSPIN_LOCK_QUEUE LockQueue
;
643 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
645 /* Get the lock index */
646 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
647 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
649 /* Now get the lock */
650 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
652 /* Acquire it and return */
653 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
659 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
661 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
663 /* Release the lock */
664 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
671 KiAcquireApcLock(IN PKTHREAD Thread
,
672 IN PKLOCK_QUEUE_HANDLE Handle
)
674 /* Acquire the lock and raise to synchronization level */
675 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
680 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
681 IN PKLOCK_QUEUE_HANDLE Handle
)
683 /* Acquire the lock */
684 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
689 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
690 IN PKLOCK_QUEUE_HANDLE Handle
)
692 /* Acquire the lock */
693 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
698 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
700 /* Release the lock */
701 KeReleaseInStackQueuedSpinLock(Handle
);
706 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
708 /* Release the lock */
709 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
714 KiAcquireProcessLock(IN PKPROCESS Process
,
715 IN PKLOCK_QUEUE_HANDLE Handle
)
717 /* Acquire the lock and raise to synchronization level */
718 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
723 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
725 /* Release the lock */
726 KeReleaseInStackQueuedSpinLock(Handle
);
731 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
733 /* Release the lock */
734 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
739 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
740 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
742 /* Check if we were called from a threaded DPC */
743 if (KeGetCurrentPrcb()->DpcThreadActive
)
745 /* Lock the Queue, we're not at DPC level */
746 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
750 /* We must be at DPC level, acquire the lock safely */
751 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
752 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
759 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
761 /* Check if we were called from a threaded DPC */
762 if (KeGetCurrentPrcb()->DpcThreadActive
)
764 /* Unlock the Queue, we're not at DPC level */
765 KeReleaseInStackQueuedSpinLock(DeviceLock
);
769 /* We must be at DPC level, release the lock safely */
770 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
771 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
776 // Satisfies the wait of any dispatcher object
778 #define KiSatisfyObjectWait(Object, Thread) \
780 /* Special case for Mutants */ \
781 if ((Object)->Header.Type == MutantObject) \
783 /* Decrease the Signal State */ \
784 (Object)->Header.SignalState--; \
786 /* Check if it's now non-signaled */ \
787 if (!(Object)->Header.SignalState) \
789 /* Set the Owner Thread */ \
790 (Object)->OwnerThread = Thread; \
792 /* Disable APCs if needed */ \
793 Thread->KernelApcDisable = Thread->KernelApcDisable - \
794 (Object)->ApcDisable; \
796 /* Check if it's abandoned */ \
797 if ((Object)->Abandoned) \
800 (Object)->Abandoned = FALSE; \
802 /* Return Status */ \
803 Thread->WaitStatus = STATUS_ABANDONED; \
806 /* Insert it into the Mutant List */ \
807 InsertHeadList(Thread->MutantListHead.Blink, \
808 &(Object)->MutantListEntry); \
811 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
812 EventSynchronizationObject) \
814 /* Synchronization Timers and Events just get un-signaled */ \
815 (Object)->Header.SignalState = 0; \
817 else if ((Object)->Header.Type == SemaphoreObject) \
819 /* These ones can have multiple states, so we only decrease it */ \
820 (Object)->Header.SignalState--; \
825 // Satisfies the wait of a mutant dispatcher object
827 #define KiSatisfyMutantWait(Object, Thread) \
829 /* Decrease the Signal State */ \
830 (Object)->Header.SignalState--; \
832 /* Check if it's now non-signaled */ \
833 if (!(Object)->Header.SignalState) \
835 /* Set the Owner Thread */ \
836 (Object)->OwnerThread = Thread; \
838 /* Disable APCs if needed */ \
839 Thread->KernelApcDisable = Thread->KernelApcDisable - \
840 (Object)->ApcDisable; \
842 /* Check if it's abandoned */ \
843 if ((Object)->Abandoned) \
846 (Object)->Abandoned = FALSE; \
848 /* Return Status */ \
849 Thread->WaitStatus = STATUS_ABANDONED; \
852 /* Insert it into the Mutant List */ \
853 InsertHeadList(Thread->MutantListHead.Blink, \
854 &(Object)->MutantListEntry); \
859 // Satisfies the wait of any nonmutant dispatcher object
861 #define KiSatisfyNonMutantWait(Object) \
863 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
864 EventSynchronizationObject) \
866 /* Synchronization Timers and Events just get un-signaled */ \
867 (Object)->Header.SignalState = 0; \
869 else if ((Object)->Header.Type == SemaphoreObject) \
871 /* These ones can have multiple states, so we only decrease it */ \
872 (Object)->Header.SignalState--; \
877 // Recalculates the due time
881 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
882 IN PLARGE_INTEGER DueTime
,
883 IN OUT PLARGE_INTEGER NewDueTime
)
885 /* Don't do anything for absolute waits */
886 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
888 /* Otherwise, query the interrupt time and recalculate */
889 NewDueTime
->QuadPart
= KeQueryInterruptTime();
890 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
895 // Determines whether a thread should be added to the wait list
899 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
900 IN KPROCESSOR_MODE WaitMode
)
902 /* Check the required conditions */
903 if ((WaitMode
!= KernelMode
) &&
904 (Thread
->EnableStackSwap
) &&
905 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
907 /* We are go for swap */
912 /* Don't swap the thread */
918 // Adds a thread to the wait list
920 #define KiAddThreadToWaitList(Thread, Swappable) \
922 /* Make sure it's swappable */ \
925 /* Insert it into the PRCB's List */ \
926 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
927 &Thread->WaitListEntry); \
932 // Checks if a wait in progress should be interrupted by APCs or an alertable
937 KiCheckAlertability(IN PKTHREAD Thread
,
938 IN BOOLEAN Alertable
,
939 IN KPROCESSOR_MODE WaitMode
)
941 /* Check if the wait is alertable */
944 /* It is, first check if the thread is alerted in this mode */
945 if (Thread
->Alerted
[WaitMode
])
947 /* It is, so bail out of the wait */
948 Thread
->Alerted
[WaitMode
] = FALSE
;
949 return STATUS_ALERTED
;
951 else if ((WaitMode
!= KernelMode
) &&
952 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
954 /* It's isn't, but this is a user wait with queued user APCs */
955 Thread
->ApcState
.UserApcPending
= TRUE
;
956 return STATUS_USER_APC
;
958 else if (Thread
->Alerted
[KernelMode
])
960 /* It isn't that either, but we're alered in kernel mode */
961 Thread
->Alerted
[KernelMode
] = FALSE
;
962 return STATUS_ALERTED
;
965 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
967 /* Not alertable, but this is a user wait with pending user APCs */
968 return STATUS_USER_APC
;
971 /* Otherwise, we're fine */
972 return STATUS_WAIT_0
;
976 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
977 // to remove timer entries
978 // See Windows HPI blog for more information.
981 KiRemoveEntryTimer(IN PKTIMER Timer
)
984 PKTIMER_TABLE_ENTRY TableEntry
;
986 /* Remove the timer from the timer list and check if it's empty */
987 Hand
= Timer
->Header
.Hand
;
988 if (RemoveEntryList(&Timer
->TimerListEntry
))
990 /* Get the respective timer table entry */
991 TableEntry
= &KiTimerTableListHead
[Hand
];
992 if (&TableEntry
->Entry
== TableEntry
->Entry
.Flink
)
994 /* Set the entry to an infinite absolute time */
995 TableEntry
->Time
.HighPart
= 0xFFFFFFFF;
999 /* Clear the list entries on dbg builds so we can tell the timer is gone */
1001 Timer
->TimerListEntry
.Flink
= NULL
;
1002 Timer
->TimerListEntry
.Blink
= NULL
;
1007 // Called by Wait and Queue code to insert a timer for dispatching.
1008 // Also called by KeSetTimerEx to insert a timer from the caller.
1012 KxInsertTimer(IN PKTIMER Timer
,
1015 PKSPIN_LOCK_QUEUE LockQueue
;
1017 /* Acquire the lock and release the dispatcher lock */
1018 LockQueue
= KiAcquireTimerLock(Hand
);
1019 KiReleaseDispatcherLockFromDpcLevel();
1021 /* Try to insert the timer */
1022 if (KiInsertTimerTable(Timer
, Hand
))
1025 KiCompleteTimer(Timer
, LockQueue
);
1029 /* Do nothing, just release the lock */
1030 KiReleaseTimerLock(LockQueue
);
1035 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
1036 // See the Windows HPI Blog for more information
1040 KiComputeDueTime(IN PKTIMER Timer
,
1041 IN LARGE_INTEGER DueTime
,
1044 LARGE_INTEGER InterruptTime
, SystemTime
, DifferenceTime
;
1046 /* Convert to relative time if needed */
1047 Timer
->Header
.Absolute
= FALSE
;
1048 if (DueTime
.HighPart
>= 0)
1050 /* Get System Time */
1051 KeQuerySystemTime(&SystemTime
);
1053 /* Do the conversion */
1054 DifferenceTime
.QuadPart
= SystemTime
.QuadPart
- DueTime
.QuadPart
;
1056 /* Make sure it hasn't already expired */
1057 Timer
->Header
.Absolute
= TRUE
;
1058 if (DifferenceTime
.HighPart
>= 0)
1060 /* Cancel everything */
1061 Timer
->Header
.SignalState
= TRUE
;
1062 Timer
->Header
.Hand
= 0;
1063 Timer
->DueTime
.QuadPart
= 0;
1068 /* Set the time as Absolute */
1069 DueTime
= DifferenceTime
;
1072 /* Get the Interrupt Time */
1073 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1075 /* Recalculate due time */
1076 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- DueTime
.QuadPart
;
1078 /* Get the handle */
1079 *Hand
= KiComputeTimerTableIndex(Timer
->DueTime
.QuadPart
);
1080 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1081 Timer
->Header
.Inserted
= TRUE
;
1086 // Called from Unlink and Queue Insert Code.
1087 // Also called by timer code when canceling an inserted timer.
1088 // Removes a timer from it's tree.
1092 KxRemoveTreeTimer(IN PKTIMER Timer
)
1094 ULONG Hand
= Timer
->Header
.Hand
;
1095 PKSPIN_LOCK_QUEUE LockQueue
;
1096 PKTIMER_TABLE_ENTRY TimerEntry
;
1098 /* Acquire timer lock */
1099 LockQueue
= KiAcquireTimerLock(Hand
);
1101 /* Set the timer as non-inserted */
1102 Timer
->Header
.Inserted
= FALSE
;
1104 /* Remove it from the timer list */
1105 if (RemoveEntryList(&Timer
->TimerListEntry
))
1107 /* Get the entry and check if it's empty */
1108 TimerEntry
= &KiTimerTableListHead
[Hand
];
1109 if (IsListEmpty(&TimerEntry
->Entry
))
1111 /* Clear the time then */
1112 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1116 /* Release the timer lock */
1117 KiReleaseTimerLock(LockQueue
);
1122 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1123 IN LARGE_INTEGER Interval
,
1127 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1129 /* Check the timer's interval to see if it's absolute */
1130 Timer
->Header
.Absolute
= FALSE
;
1131 if (Interval
.HighPart
>= 0)
1133 /* Get the system time and calculate the relative time */
1134 KeQuerySystemTime(&SystemTime
);
1135 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1136 Timer
->Header
.Absolute
= TRUE
;
1138 /* Check if we've already expired */
1139 if (TimeDifference
.HighPart
>= 0)
1141 /* Reset everything */
1142 Timer
->DueTime
.QuadPart
= 0;
1144 Timer
->Header
.Hand
= 0;
1149 /* Update the interval */
1150 Interval
= TimeDifference
;
1154 /* Calculate the due time */
1155 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1156 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1157 Timer
->DueTime
.QuadPart
= DueTime
;
1159 /* Calculate the timer handle */
1160 *Hand
= KiComputeTimerTableIndex(DueTime
);
1161 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1164 #define KxDelayThreadWait() \
1166 /* Setup the Wait Block */ \
1167 Thread->WaitBlockList = TimerBlock; \
1169 /* Setup the timer */ \
1170 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1172 /* Save the due time for the caller */ \
1173 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1175 /* Link the timer to this Wait Block */ \
1176 TimerBlock->NextWaitBlock = TimerBlock; \
1177 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1178 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1180 /* Clear wait status */ \
1181 Thread->WaitStatus = STATUS_SUCCESS; \
1183 /* Setup wait fields */ \
1184 Thread->Alertable = Alertable; \
1185 Thread->WaitReason = DelayExecution; \
1186 Thread->WaitMode = WaitMode; \
1188 /* Check if we can swap the thread's stack */ \
1189 Thread->WaitListEntry.Flink = NULL; \
1190 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1192 /* Set the wait time */ \
1193 Thread->WaitTime = KeTickCount.LowPart;
1195 #define KxMultiThreadWait() \
1196 /* Link wait block array to the thread */ \
1197 Thread->WaitBlockList = WaitBlockArray; \
1199 /* Reset the index */ \
1202 /* Loop wait blocks */ \
1205 /* Fill out the wait block */ \
1206 WaitBlock = &WaitBlockArray[Index]; \
1207 WaitBlock->Object = Object[Index]; \
1208 WaitBlock->WaitKey = (USHORT)Index; \
1209 WaitBlock->WaitType = WaitType; \
1210 WaitBlock->Thread = Thread; \
1212 /* Link to next block */ \
1213 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1215 } while (Index < Count); \
1217 /* Link the last block */ \
1218 WaitBlock->NextWaitBlock = WaitBlockArray; \
1220 /* Set default wait status */ \
1221 Thread->WaitStatus = STATUS_WAIT_0; \
1223 /* Check if we have a timer */ \
1226 /* Link to the block */ \
1227 TimerBlock->NextWaitBlock = WaitBlockArray; \
1229 /* Setup the timer */ \
1230 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1232 /* Save the due time for the caller */ \
1233 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1235 /* Initialize the list */ \
1236 InitializeListHead(&Timer->Header.WaitListHead); \
1239 /* Set wait settings */ \
1240 Thread->Alertable = Alertable; \
1241 Thread->WaitMode = WaitMode; \
1242 Thread->WaitReason = WaitReason; \
1244 /* Check if we can swap the thread's stack */ \
1245 Thread->WaitListEntry.Flink = NULL; \
1246 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1248 /* Set the wait time */ \
1249 Thread->WaitTime = KeTickCount.LowPart;
1251 #define KxSingleThreadWait() \
1252 /* Setup the Wait Block */ \
1253 Thread->WaitBlockList = WaitBlock; \
1254 WaitBlock->WaitKey = STATUS_SUCCESS; \
1255 WaitBlock->Object = Object; \
1256 WaitBlock->WaitType = WaitAny; \
1258 /* Clear wait status */ \
1259 Thread->WaitStatus = STATUS_SUCCESS; \
1261 /* Check if we have a timer */ \
1264 /* Setup the timer */ \
1265 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1267 /* Save the due time for the caller */ \
1268 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1270 /* Pointer to timer block */ \
1271 WaitBlock->NextWaitBlock = TimerBlock; \
1272 TimerBlock->NextWaitBlock = WaitBlock; \
1274 /* Link the timer to this Wait Block */ \
1275 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1276 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1280 /* No timer block, just ourselves */ \
1281 WaitBlock->NextWaitBlock = WaitBlock; \
1284 /* Set wait settings */ \
1285 Thread->Alertable = Alertable; \
1286 Thread->WaitMode = WaitMode; \
1287 Thread->WaitReason = WaitReason; \
1289 /* Check if we can swap the thread's stack */ \
1290 Thread->WaitListEntry.Flink = NULL; \
1291 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1293 /* Set the wait time */ \
1294 Thread->WaitTime = KeTickCount.LowPart;
1296 #define KxQueueThreadWait() \
1297 /* Setup the Wait Block */ \
1298 Thread->WaitBlockList = WaitBlock; \
1299 WaitBlock->WaitKey = STATUS_SUCCESS; \
1300 WaitBlock->Object = Queue; \
1301 WaitBlock->WaitType = WaitAny; \
1302 WaitBlock->Thread = Thread; \
1304 /* Clear wait status */ \
1305 Thread->WaitStatus = STATUS_SUCCESS; \
1307 /* Check if we have a timer */ \
1310 /* Setup the timer */ \
1311 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1313 /* Save the due time for the caller */ \
1314 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1316 /* Pointer to timer block */ \
1317 WaitBlock->NextWaitBlock = TimerBlock; \
1318 TimerBlock->NextWaitBlock = WaitBlock; \
1320 /* Link the timer to this Wait Block */ \
1321 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1322 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1326 /* No timer block, just ourselves */ \
1327 WaitBlock->NextWaitBlock = WaitBlock; \
1330 /* Set wait settings */ \
1331 Thread->Alertable = FALSE; \
1332 Thread->WaitMode = WaitMode; \
1333 Thread->WaitReason = WrQueue; \
1335 /* Check if we can swap the thread's stack */ \
1336 Thread->WaitListEntry.Flink = NULL; \
1337 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1339 /* Set the wait time */ \
1340 Thread->WaitTime = KeTickCount.LowPart;
1347 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1348 IN KPRIORITY Increment
)
1350 PLIST_ENTRY WaitEntry
, WaitList
;
1351 PKWAIT_BLOCK WaitBlock
;
1352 PKTHREAD WaitThread
;
1355 /* Loop the Wait Entries */
1356 WaitList
= &Object
->WaitListHead
;
1357 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1358 WaitEntry
= WaitList
->Flink
;
1361 /* Get the current wait block */
1362 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1364 /* Get the waiting thread */
1365 WaitThread
= WaitBlock
->Thread
;
1367 /* Check the current Wait Mode */
1368 if (WaitBlock
->WaitType
== WaitAny
)
1370 /* Use the actual wait key */
1371 WaitKey
= WaitBlock
->WaitKey
;
1375 /* Otherwise, use STATUS_KERNEL_APC */
1376 WaitKey
= STATUS_KERNEL_APC
;
1379 /* Unwait the thread */
1380 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1383 WaitEntry
= WaitList
->Flink
;
1384 } while (WaitEntry
!= WaitList
);
1388 // Unwaits a Thread waiting on an event
1392 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1393 IN KPRIORITY Increment
)
1395 PLIST_ENTRY WaitEntry
, WaitList
;
1396 PKWAIT_BLOCK WaitBlock
;
1397 PKTHREAD WaitThread
;
1399 /* Loop the Wait Entries */
1400 WaitList
= &Event
->Header
.WaitListHead
;
1401 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1402 WaitEntry
= WaitList
->Flink
;
1405 /* Get the current wait block */
1406 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1408 /* Get the waiting thread */
1409 WaitThread
= WaitBlock
->Thread
;
1411 /* Check the current Wait Mode */
1412 if (WaitBlock
->WaitType
== WaitAny
)
1415 Event
->Header
.SignalState
= 0;
1417 /* Un-signal the event and unwait the thread */
1418 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1422 /* Unwait the thread with STATUS_KERNEL_APC */
1423 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1426 WaitEntry
= WaitList
->Flink
;
1427 } while (WaitEntry
!= WaitList
);
1431 // This routine queues a thread that is ready on the PRCB's ready lists.
1432 // If this thread cannot currently run on this CPU, then the thread is
1433 // added to the deferred ready list instead.
1435 // This routine must be entered with the PRCB lock held and it will exit
1436 // with the PRCB lock released!
1440 KxQueueReadyThread(IN PKTHREAD Thread
,
1447 ASSERT(Prcb
== KeGetCurrentPrcb());
1448 ASSERT(Thread
->State
== Running
);
1449 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1451 /* Check if this thread is allowed to run in this CPU */
1453 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1458 /* Set thread ready for execution */
1459 Thread
->State
= Ready
;
1461 /* Save current priority and if someone had pre-empted it */
1462 Priority
= Thread
->Priority
;
1463 Preempted
= Thread
->Preempted
;
1465 /* We're not pre-empting now, and set the wait time */
1466 Thread
->Preempted
= FALSE
;
1467 Thread
->WaitTime
= KeTickCount
.LowPart
;
1470 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1472 /* Insert this thread in the appropriate order */
1473 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1474 &Thread
->WaitListEntry
) :
1475 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1476 &Thread
->WaitListEntry
);
1478 /* Update the ready summary */
1479 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1482 ASSERT(Priority
== Thread
->Priority
);
1484 /* Release the PRCB lock */
1485 KiReleasePrcbLock(Prcb
);
1489 /* Otherwise, prepare this thread to be deferred */
1490 Thread
->State
= DeferredReady
;
1491 Thread
->DeferredProcessor
= Prcb
->Number
;
1493 /* Release the lock and defer scheduling */
1494 KiReleasePrcbLock(Prcb
);
1495 KiDeferredReadyThread(Thread
);
1500 // This routine scans for an appropriate ready thread to select at the
1501 // given priority and for the given CPU.
1505 KiSelectReadyThread(IN KPRIORITY Priority
,
1510 PLIST_ENTRY ListEntry
;
1511 PKTHREAD Thread
= NULL
;
1513 /* Save the current mask and get the priority set for the CPU */
1514 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1515 if (!PrioritySet
) goto Quickie
;
1517 /* Get the highest priority possible */
1518 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1519 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1520 HighPriority
+= Priority
;
1522 /* Make sure the list isn't empty at the highest priority */
1523 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1525 /* Get the first thread on the list */
1526 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1527 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1529 /* Make sure this thread is here for a reason */
1530 ASSERT(HighPriority
== Thread
->Priority
);
1531 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1532 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1534 /* Remove it from the list */
1535 if (RemoveEntryList(&Thread
->WaitListEntry
))
1537 /* The list is empty now, reset the ready summary */
1538 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1541 /* Sanity check and return the thread */
1543 ASSERT((Thread
== NULL
) ||
1544 (Thread
->BasePriority
== 0) ||
1545 (Thread
->Priority
!= 0));
1550 // This routine computes the new priority for a thread. It is only valid for
1551 // threads with priorities in the dynamic priority range.
1555 KiComputeNewPriority(IN PKTHREAD Thread
,
1556 IN SCHAR Adjustment
)
1560 /* Priority sanity checks */
1561 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1562 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1563 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1564 TRUE
: (Thread
->PriorityDecrement
== 0));
1566 /* Get the current priority */
1567 Priority
= Thread
->Priority
;
1568 if (Priority
< LOW_REALTIME_PRIORITY
)
1570 /* Decrease priority by the priority decrement */
1571 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1573 /* Don't go out of bounds */
1574 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1576 /* Reset the priority decrement */
1577 Thread
->PriorityDecrement
= 0;
1581 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1583 /* Return the new priority */
1588 // Guarded Mutex Routines
1592 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex
)
1594 /* Setup the Initial Data */
1595 GuardedMutex
->Count
= GM_LOCK_BIT
;
1596 GuardedMutex
->Owner
= NULL
;
1597 GuardedMutex
->Contention
= 0;
1599 /* Initialize the Wait Gate */
1600 KeInitializeGate(&GuardedMutex
->Gate
);
1605 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1607 PKTHREAD Thread
= KeGetCurrentThread();
1610 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1611 (Thread
->SpecialApcDisable
< 0) ||
1612 (Thread
->Teb
== NULL
) ||
1613 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1614 ASSERT(GuardedMutex
->Owner
!= Thread
);
1616 /* Remove the lock */
1617 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1619 /* The Guarded Mutex was already locked, enter contented case */
1620 KiAcquireGuardedMutex(GuardedMutex
);
1624 GuardedMutex
->Owner
= Thread
;
1629 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1631 LONG OldValue
, NewValue
;
1634 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1635 (KeGetCurrentThread()->SpecialApcDisable
< 0) ||
1636 (KeGetCurrentThread()->Teb
== NULL
) ||
1637 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1638 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1640 /* Destroy the Owner */
1641 GuardedMutex
->Owner
= NULL
;
1643 /* Add the Lock Bit */
1644 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1645 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1647 /* Check if it was already locked, but not woken */
1648 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1650 /* Update the Oldvalue to what it should be now */
1651 OldValue
+= GM_LOCK_BIT
;
1653 /* The mutex will be woken, minus one waiter */
1654 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1657 /* Remove the Woken bit */
1658 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1660 OldValue
) == OldValue
)
1662 /* Signal the Gate */
1663 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1670 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex
)
1672 PKTHREAD Thread
= KeGetCurrentThread();
1675 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1676 ASSERT(GuardedMutex
->Owner
!= Thread
);
1678 /* Disable Special APCs */
1679 KeEnterGuardedRegion();
1681 /* Remove the lock */
1682 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1684 /* The Guarded Mutex was already locked, enter contented case */
1685 KiAcquireGuardedMutex(GuardedMutex
);
1688 /* Set the Owner and Special APC Disable state */
1689 GuardedMutex
->Owner
= Thread
;
1690 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;
1695 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1697 LONG OldValue
, NewValue
;
1700 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1701 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1702 ASSERT(KeGetCurrentThread()->SpecialApcDisable
==
1703 GuardedMutex
->SpecialApcDisable
);
1705 /* Destroy the Owner */
1706 GuardedMutex
->Owner
= NULL
;
1708 /* Add the Lock Bit */
1709 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1710 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1712 /* Check if it was already locked, but not woken */
1713 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1715 /* Update the Oldvalue to what it should be now */
1716 OldValue
+= GM_LOCK_BIT
;
1718 /* The mutex will be woken, minus one waiter */
1719 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1722 /* Remove the Woken bit */
1723 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1725 OldValue
) == OldValue
)
1727 /* Signal the Gate */
1728 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1732 /* Re-enable APCs */
1733 KeLeaveGuardedRegion();
1738 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1740 PKTHREAD Thread
= KeGetCurrentThread();
1743 KeEnterGuardedRegion();
1745 /* Remove the lock */
1746 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1748 /* Re-enable APCs */
1749 KeLeaveGuardedRegion();
1752 /* Return failure */
1756 /* Set the Owner and APC State */
1757 GuardedMutex
->Owner
= Thread
;
1758 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;