2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
12 KeGetCurrentThread(VOID
)
15 /* Return the current thread */
16 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
17 #elif defined (_M_AMD64)
18 return (PRKTHREAD
)__readgsqword(FIELD_OFFSET(KIPCR
, Prcb
.CurrentThread
));
20 PKPRCB Prcb
= KeGetCurrentPrcb();
21 return Prcb
->CurrentThread
;
27 KeGetPreviousMode(VOID
)
29 /* Return the current mode */
30 return KeGetCurrentThread()->PreviousMode
;
36 KeFlushProcessTb(VOID
)
38 /* Flush the TLB by resetting CR3 */
40 __asm__("sync\n\tisync\n\t");
43 // We need to implement this!
45 ASSERTMSG("Need ARM flush routine\n", FALSE
);
47 __writecr3(__readcr3());
52 // Enters a Guarded Region
54 #define KeEnterGuardedRegion() \
56 PKTHREAD _Thread = KeGetCurrentThread(); \
59 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
60 ASSERT(_Thread == KeGetCurrentThread()); \
61 ASSERT((_Thread->SpecialApcDisable <= 0) && \
62 (_Thread->SpecialApcDisable != -32768)); \
64 /* Disable Special APCs */ \
65 _Thread->SpecialApcDisable--; \
69 // Leaves a Guarded Region
71 #define KeLeaveGuardedRegion() \
73 PKTHREAD _Thread = KeGetCurrentThread(); \
76 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
77 ASSERT(_Thread == KeGetCurrentThread()); \
78 ASSERT(_Thread->SpecialApcDisable < 0); \
80 /* Leave region and check if APCs are OK now */ \
81 if (!(++_Thread->SpecialApcDisable)) \
83 /* Check for Kernel APCs on the list */ \
84 if (!IsListEmpty(&_Thread->ApcState. \
85 ApcListHead[KernelMode])) \
87 /* Check for APC Delivery */ \
88 KiCheckForKernelApcDelivery(); \
94 // Enters a Critical Region
96 #define KeEnterCriticalRegion() \
98 PKTHREAD _Thread = KeGetCurrentThread(); \
100 /* Sanity checks */ \
101 ASSERT(_Thread == KeGetCurrentThread()); \
102 ASSERT((_Thread->KernelApcDisable <= 0) && \
103 (_Thread->KernelApcDisable != -32768)); \
105 /* Disable Kernel APCs */ \
106 _Thread->KernelApcDisable--; \
110 // Leaves a Critical Region
112 #define KeLeaveCriticalRegion() \
114 PKTHREAD _Thread = KeGetCurrentThread(); \
116 /* Sanity checks */ \
117 ASSERT(_Thread == KeGetCurrentThread()); \
118 ASSERT(_Thread->KernelApcDisable < 0); \
120 /* Enable Kernel APCs */ \
121 _Thread->KernelApcDisable++; \
123 /* Check if Kernel APCs are now enabled */ \
124 if (!(_Thread->KernelApcDisable)) \
126 /* Check if we need to request an APC Delivery */ \
127 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
128 !(_Thread->SpecialApcDisable)) \
130 /* Check for the right environment */ \
131 KiCheckForKernelApcDelivery(); \
138 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
142 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
144 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
145 UNREFERENCED_PARAMETER(SpinLock
);
149 // Spinlock Release at IRQL >= DISPATCH_LEVEL
153 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
155 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
156 UNREFERENCED_PARAMETER(SpinLock
);
160 // This routine protects against multiple CPU acquires, it's meaningless on UP.
164 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
166 UNREFERENCED_PARAMETER(Object
);
170 // This routine protects against multiple CPU acquires, it's meaningless on UP.
174 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
176 UNREFERENCED_PARAMETER(Object
);
181 KiAcquireDispatcherLock(VOID
)
183 /* Raise to DPC level */
184 return KeRaiseIrqlToDpcLevel();
189 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
191 /* Just exit the dispatcher */
192 KiExitDispatcher(OldIrql
);
197 KiAcquireDispatcherLockAtDpcLevel(VOID
)
199 /* This is a no-op at DPC Level for UP systems */
205 KiReleaseDispatcherLockFromDpcLevel(VOID
)
207 /* This is a no-op at DPC Level for UP systems */
212 // This routine makes the thread deferred ready on the boot CPU.
216 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
218 /* Set the thread to deferred state and boot CPU */
219 Thread
->State
= DeferredReady
;
220 Thread
->DeferredProcessor
= 0;
222 /* Make the thread ready immediately */
223 KiDeferredReadyThread(Thread
);
228 KiRescheduleThread(IN BOOLEAN NewThread
,
231 /* This is meaningless on UP systems */
232 UNREFERENCED_PARAMETER(NewThread
);
233 UNREFERENCED_PARAMETER(Cpu
);
237 // This routine protects against multiple CPU acquires, it's meaningless on UP.
241 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
243 UNREFERENCED_PARAMETER(Thread
);
247 // This routine protects against multiple CPU acquires, it's meaningless on UP.
251 KiAcquirePrcbLock(IN PKPRCB Prcb
)
253 UNREFERENCED_PARAMETER(Prcb
);
257 // This routine protects against multiple CPU acquires, it's meaningless on UP.
261 KiReleasePrcbLock(IN PKPRCB Prcb
)
263 UNREFERENCED_PARAMETER(Prcb
);
267 // This routine protects against multiple CPU acquires, it's meaningless on UP.
271 KiAcquireThreadLock(IN PKTHREAD Thread
)
273 UNREFERENCED_PARAMETER(Thread
);
277 // This routine protects against multiple CPU acquires, it's meaningless on UP.
281 KiReleaseThreadLock(IN PKTHREAD Thread
)
283 UNREFERENCED_PARAMETER(Thread
);
287 // This routine protects against multiple CPU acquires, it's meaningless on UP.
291 KiTryThreadLock(IN PKTHREAD Thread
)
293 UNREFERENCED_PARAMETER(Thread
);
299 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
301 /* There are no deferred ready lists on UP systems */
302 UNREFERENCED_PARAMETER(Prcb
);
307 KiRundownThread(IN PKTHREAD Thread
)
310 /* Check if this is the NPX Thread */
311 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
314 KeGetCurrentPrcb()->NpxThread
= NULL
;
322 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
325 /* We deliver instantly on UP */
326 UNREFERENCED_PARAMETER(NeedApc
);
327 UNREFERENCED_PARAMETER(Processor
);
332 KiAcquireTimerLock(IN ULONG Hand
)
334 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
336 /* Nothing to do on UP */
337 UNREFERENCED_PARAMETER(Hand
);
343 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
345 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
347 /* Nothing to do on UP */
348 UNREFERENCED_PARAMETER(LockQueue
);
354 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
358 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
360 /* Make sure that we don't own the lock already */
361 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) == *SpinLock
)
363 /* We do, bugcheck! */
364 KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
367 /* Start acquire loop */
370 /* Try to acquire it */
371 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
373 /* Value changed... wait until it's unlocked */
374 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
377 /* On debug builds, we use a much slower but useful routine */
378 //Kii386SpinOnSpinLock(SpinLock, 5);
380 /* FIXME: Do normal yield for now */
383 /* Otherwise, just yield and keep looping */
391 /* On debug builds, we OR in the KTHREAD */
392 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
394 /* All is well, break out */
401 // Spinlock Release at IRQL >= DISPATCH_LEVEL
405 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
408 /* Make sure that the threads match */
409 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
411 /* They don't, bugcheck */
412 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
416 InterlockedAnd((PLONG
)SpinLock
, 0);
421 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
425 /* Make sure we're at a safe level to touch the lock */
426 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
428 /* Start acquire loop */
431 /* Loop until the other CPU releases it */
434 /* Check if it got released */
435 OldValue
= Object
->Lock
;
436 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
438 /* Let the CPU know that this is a loop */
442 /* Try acquiring the lock now */
443 } while (InterlockedCompareExchange(&Object
->Lock
,
444 OldValue
| KOBJECT_LOCK_BIT
,
445 OldValue
) != OldValue
);
450 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
452 /* Make sure we're at a safe level to touch the lock */
453 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
456 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
461 KiAcquireDispatcherLock(VOID
)
463 /* Raise to synchronization level and acquire the dispatcher lock */
464 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
469 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
471 /* First release the lock */
472 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
473 LockQueue
[LockQueueDispatcherLock
]);
475 /* Then exit the dispatcher */
476 KiExitDispatcher(OldIrql
);
481 KiAcquireDispatcherLockAtDpcLevel(VOID
)
483 /* Acquire the dispatcher lock */
484 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
485 LockQueue
[LockQueueDispatcherLock
]);
490 KiReleaseDispatcherLockFromDpcLevel(VOID
)
492 /* Release the dispatcher lock */
493 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
494 LockQueue
[LockQueueDispatcherLock
]);
498 // This routine inserts a thread into the deferred ready list of the current CPU
502 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
504 PKPRCB Prcb
= KeGetCurrentPrcb();
506 /* Set the thread to deferred state and CPU */
507 Thread
->State
= DeferredReady
;
508 Thread
->DeferredProcessor
= Prcb
->Number
;
510 /* Add it on the list */
511 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
516 KiRescheduleThread(IN BOOLEAN NewThread
,
519 /* Check if a new thread needs to be scheduled on a different CPU */
520 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
522 /* Send an IPI to request delivery */
523 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
528 // This routine sets the current thread in a swap busy state, which ensure that
529 // nobody else tries to swap it concurrently.
533 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
535 /* Make sure nobody already set it */
536 ASSERT(Thread
->SwapBusy
== FALSE
);
538 /* Set it ourselves */
539 Thread
->SwapBusy
= TRUE
;
543 // This routine acquires the PRCB lock so that only one caller can touch
544 // volatile PRCB data.
546 // Since this is a simple optimized spin-lock, it must only be acquired
547 // at dispatcher level or higher!
551 KiAcquirePrcbLock(IN PKPRCB Prcb
)
553 /* Make sure we're at a safe level to touch the PRCB lock */
554 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
556 /* Start acquire loop */
559 /* Acquire the lock and break out if we acquired it first */
560 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
562 /* Loop until the other CPU releases it */
565 /* Let the CPU know that this is a loop */
567 } while (Prcb
->PrcbLock
);
572 // This routine releases the PRCB lock so that other callers can touch
573 // volatile PRCB data.
575 // Since this is a simple optimized spin-lock, it must be be only acquired
576 // at dispatcher level or higher!
580 KiReleasePrcbLock(IN PKPRCB Prcb
)
582 /* Make sure we are above dispatch and the lock is acquired! */
583 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
584 ASSERT(Prcb
->PrcbLock
!= 0);
587 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
591 // This routine acquires the thread lock so that only one caller can touch
592 // volatile thread data.
594 // Since this is a simple optimized spin-lock, it must be be only acquired
595 // at dispatcher level or higher!
599 KiAcquireThreadLock(IN PKTHREAD Thread
)
601 /* Make sure we're at a safe level to touch the thread lock */
602 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
604 /* Start acquire loop */
607 /* Acquire the lock and break out if we acquired it first */
608 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
610 /* Loop until the other CPU releases it */
613 /* Let the CPU know that this is a loop */
615 } while (Thread
->ThreadLock
);
620 // This routine releases the thread lock so that other callers can touch
621 // volatile thread data.
623 // Since this is a simple optimized spin-lock, it must be be only acquired
624 // at dispatcher level or higher!
628 KiReleaseThreadLock(IN PKTHREAD Thread
)
630 /* Make sure we are still above dispatch */
631 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
634 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
639 KiTryThreadLock(IN PKTHREAD Thread
)
643 /* If the lock isn't acquired, return false */
644 if (!Thread
->ThreadLock
) return FALSE
;
646 /* Otherwise, try to acquire it and check the result */
648 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
650 /* Return the lock state */
651 return (Value
== TRUE
);
656 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
658 /* Scan the deferred ready lists if required */
659 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
664 KiRundownThread(IN PKTHREAD Thread
)
672 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
675 /* Check if we need to request APC delivery */
678 /* Check if it's on another CPU */
679 if (KeGetPcr()->Number
!= Processor
)
681 /* Send an IPI to request delivery */
682 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
686 /* Request a software interrupt */
687 HalRequestSoftwareInterrupt(APC_LEVEL
);
694 KiAcquireTimerLock(IN ULONG Hand
)
696 PKSPIN_LOCK_QUEUE LockQueue
;
698 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
700 /* Get the lock index */
701 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
702 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
704 /* Now get the lock */
705 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
707 /* Acquire it and return */
708 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
714 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
716 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
718 /* Release the lock */
719 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
726 KiAcquireApcLock(IN PKTHREAD Thread
,
727 IN PKLOCK_QUEUE_HANDLE Handle
)
729 /* Acquire the lock and raise to synchronization level */
730 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
735 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
736 IN PKLOCK_QUEUE_HANDLE Handle
)
738 /* Acquire the lock */
739 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
744 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
745 IN PKLOCK_QUEUE_HANDLE Handle
)
747 /* Acquire the lock */
748 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
753 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
755 /* Release the lock */
756 KeReleaseInStackQueuedSpinLock(Handle
);
761 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
763 /* Release the lock */
764 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
769 KiAcquireProcessLock(IN PKPROCESS Process
,
770 IN PKLOCK_QUEUE_HANDLE Handle
)
772 /* Acquire the lock and raise to synchronization level */
773 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
778 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
780 /* Release the lock */
781 KeReleaseInStackQueuedSpinLock(Handle
);
786 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
788 /* Release the lock */
789 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
794 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
795 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
797 /* Check if we were called from a threaded DPC */
798 if (KeGetCurrentPrcb()->DpcThreadActive
)
800 /* Lock the Queue, we're not at DPC level */
801 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
805 /* We must be at DPC level, acquire the lock safely */
806 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
807 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
814 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
816 /* Check if we were called from a threaded DPC */
817 if (KeGetCurrentPrcb()->DpcThreadActive
)
819 /* Unlock the Queue, we're not at DPC level */
820 KeReleaseInStackQueuedSpinLock(DeviceLock
);
824 /* We must be at DPC level, release the lock safely */
825 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
826 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
831 // Satisfies the wait of any dispatcher object
833 #define KiSatisfyObjectWait(Object, Thread) \
835 /* Special case for Mutants */ \
836 if ((Object)->Header.Type == MutantObject) \
838 /* Decrease the Signal State */ \
839 (Object)->Header.SignalState--; \
841 /* Check if it's now non-signaled */ \
842 if (!(Object)->Header.SignalState) \
844 /* Set the Owner Thread */ \
845 (Object)->OwnerThread = Thread; \
847 /* Disable APCs if needed */ \
848 Thread->KernelApcDisable = Thread->KernelApcDisable - \
849 (Object)->ApcDisable; \
851 /* Check if it's abandoned */ \
852 if ((Object)->Abandoned) \
855 (Object)->Abandoned = FALSE; \
857 /* Return Status */ \
858 Thread->WaitStatus = STATUS_ABANDONED; \
861 /* Insert it into the Mutant List */ \
862 InsertHeadList(Thread->MutantListHead.Blink, \
863 &(Object)->MutantListEntry); \
866 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
867 EventSynchronizationObject) \
869 /* Synchronization Timers and Events just get un-signaled */ \
870 (Object)->Header.SignalState = 0; \
872 else if ((Object)->Header.Type == SemaphoreObject) \
874 /* These ones can have multiple states, so we only decrease it */ \
875 (Object)->Header.SignalState--; \
880 // Satisfies the wait of a mutant dispatcher object
882 #define KiSatisfyMutantWait(Object, Thread) \
884 /* Decrease the Signal State */ \
885 (Object)->Header.SignalState--; \
887 /* Check if it's now non-signaled */ \
888 if (!(Object)->Header.SignalState) \
890 /* Set the Owner Thread */ \
891 (Object)->OwnerThread = Thread; \
893 /* Disable APCs if needed */ \
894 Thread->KernelApcDisable = Thread->KernelApcDisable - \
895 (Object)->ApcDisable; \
897 /* Check if it's abandoned */ \
898 if ((Object)->Abandoned) \
901 (Object)->Abandoned = FALSE; \
903 /* Return Status */ \
904 Thread->WaitStatus = STATUS_ABANDONED; \
907 /* Insert it into the Mutant List */ \
908 InsertHeadList(Thread->MutantListHead.Blink, \
909 &(Object)->MutantListEntry); \
914 // Satisfies the wait of any nonmutant dispatcher object
916 #define KiSatisfyNonMutantWait(Object) \
918 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
919 EventSynchronizationObject) \
921 /* Synchronization Timers and Events just get un-signaled */ \
922 (Object)->Header.SignalState = 0; \
924 else if ((Object)->Header.Type == SemaphoreObject) \
926 /* These ones can have multiple states, so we only decrease it */ \
927 (Object)->Header.SignalState--; \
932 // Recalculates the due time
936 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
937 IN PLARGE_INTEGER DueTime
,
938 IN OUT PLARGE_INTEGER NewDueTime
)
940 /* Don't do anything for absolute waits */
941 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
943 /* Otherwise, query the interrupt time and recalculate */
944 NewDueTime
->QuadPart
= KeQueryInterruptTime();
945 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
950 // Determines whether a thread should be added to the wait list
954 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
955 IN KPROCESSOR_MODE WaitMode
)
957 /* Check the required conditions */
958 if ((WaitMode
!= KernelMode
) &&
959 (Thread
->EnableStackSwap
) &&
960 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
962 /* We are go for swap */
967 /* Don't swap the thread */
973 // Adds a thread to the wait list
975 #define KiAddThreadToWaitList(Thread, Swappable) \
977 /* Make sure it's swappable */ \
980 /* Insert it into the PRCB's List */ \
981 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
982 &Thread->WaitListEntry); \
987 // Checks if a wait in progress should be interrupted by APCs or an alertable
992 KiCheckAlertability(IN PKTHREAD Thread
,
993 IN BOOLEAN Alertable
,
994 IN KPROCESSOR_MODE WaitMode
)
996 /* Check if the wait is alertable */
999 /* It is, first check if the thread is alerted in this mode */
1000 if (Thread
->Alerted
[WaitMode
])
1002 /* It is, so bail out of the wait */
1003 Thread
->Alerted
[WaitMode
] = FALSE
;
1004 return STATUS_ALERTED
;
1006 else if ((WaitMode
!= KernelMode
) &&
1007 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
1009 /* It's isn't, but this is a user wait with queued user APCs */
1010 Thread
->ApcState
.UserApcPending
= TRUE
;
1011 return STATUS_USER_APC
;
1013 else if (Thread
->Alerted
[KernelMode
])
1015 /* It isn't that either, but we're alered in kernel mode */
1016 Thread
->Alerted
[KernelMode
] = FALSE
;
1017 return STATUS_ALERTED
;
1020 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
1022 /* Not alertable, but this is a user wait with pending user APCs */
1023 return STATUS_USER_APC
;
1026 /* Otherwise, we're fine */
1027 return STATUS_WAIT_0
;
1031 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
1032 // to remove timer entries
1033 // See Windows HPI blog for more information.
1036 KiRemoveEntryTimer(IN PKTIMER Timer
)
1039 PKTIMER_TABLE_ENTRY TableEntry
;
1041 /* Remove the timer from the timer list and check if it's empty */
1042 Hand
= Timer
->Header
.Hand
;
1043 if (RemoveEntryList(&Timer
->TimerListEntry
))
1045 /* Get the respective timer table entry */
1046 TableEntry
= &KiTimerTableListHead
[Hand
];
1047 if (&TableEntry
->Entry
== TableEntry
->Entry
.Flink
)
1049 /* Set the entry to an infinite absolute time */
1050 TableEntry
->Time
.HighPart
= 0xFFFFFFFF;
1054 /* Clear the list entries on dbg builds so we can tell the timer is gone */
1056 Timer
->TimerListEntry
.Flink
= NULL
;
1057 Timer
->TimerListEntry
.Blink
= NULL
;
1062 // Called by Wait and Queue code to insert a timer for dispatching.
1063 // Also called by KeSetTimerEx to insert a timer from the caller.
1067 KxInsertTimer(IN PKTIMER Timer
,
1070 PKSPIN_LOCK_QUEUE LockQueue
;
1072 /* Acquire the lock and release the dispatcher lock */
1073 LockQueue
= KiAcquireTimerLock(Hand
);
1074 KiReleaseDispatcherLockFromDpcLevel();
1076 /* Try to insert the timer */
1077 if (KiInsertTimerTable(Timer
, Hand
))
1080 KiCompleteTimer(Timer
, LockQueue
);
1084 /* Do nothing, just release the lock */
1085 KiReleaseTimerLock(LockQueue
);
1090 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
1091 // See the Windows HPI Blog for more information
1095 KiComputeDueTime(IN PKTIMER Timer
,
1096 IN LARGE_INTEGER DueTime
,
1099 LARGE_INTEGER InterruptTime
, SystemTime
, DifferenceTime
;
1101 /* Convert to relative time if needed */
1102 Timer
->Header
.Absolute
= FALSE
;
1103 if (DueTime
.HighPart
>= 0)
1105 /* Get System Time */
1106 KeQuerySystemTime(&SystemTime
);
1108 /* Do the conversion */
1109 DifferenceTime
.QuadPart
= SystemTime
.QuadPart
- DueTime
.QuadPart
;
1111 /* Make sure it hasn't already expired */
1112 Timer
->Header
.Absolute
= TRUE
;
1113 if (DifferenceTime
.HighPart
>= 0)
1115 /* Cancel everything */
1116 Timer
->Header
.SignalState
= TRUE
;
1117 Timer
->Header
.Hand
= 0;
1118 Timer
->DueTime
.QuadPart
= 0;
1123 /* Set the time as Absolute */
1124 DueTime
= DifferenceTime
;
1127 /* Get the Interrupt Time */
1128 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1130 /* Recalculate due time */
1131 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- DueTime
.QuadPart
;
1133 /* Get the handle */
1134 *Hand
= KiComputeTimerTableIndex(Timer
->DueTime
.QuadPart
);
1135 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1136 Timer
->Header
.Inserted
= TRUE
;
1141 // Called from Unlink and Queue Insert Code.
1142 // Also called by timer code when canceling an inserted timer.
1143 // Removes a timer from it's tree.
1147 KxRemoveTreeTimer(IN PKTIMER Timer
)
1149 ULONG Hand
= Timer
->Header
.Hand
;
1150 PKSPIN_LOCK_QUEUE LockQueue
;
1151 PKTIMER_TABLE_ENTRY TimerEntry
;
1153 /* Acquire timer lock */
1154 LockQueue
= KiAcquireTimerLock(Hand
);
1156 /* Set the timer as non-inserted */
1157 Timer
->Header
.Inserted
= FALSE
;
1159 /* Remove it from the timer list */
1160 if (RemoveEntryList(&Timer
->TimerListEntry
))
1162 /* Get the entry and check if it's empty */
1163 TimerEntry
= &KiTimerTableListHead
[Hand
];
1164 if (IsListEmpty(&TimerEntry
->Entry
))
1166 /* Clear the time then */
1167 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1171 /* Release the timer lock */
1172 KiReleaseTimerLock(LockQueue
);
1177 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1178 IN LARGE_INTEGER Interval
,
1182 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1184 /* Check the timer's interval to see if it's absolute */
1185 Timer
->Header
.Absolute
= FALSE
;
1186 if (Interval
.HighPart
>= 0)
1188 /* Get the system time and calculate the relative time */
1189 KeQuerySystemTime(&SystemTime
);
1190 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1191 Timer
->Header
.Absolute
= TRUE
;
1193 /* Check if we've already expired */
1194 if (TimeDifference
.HighPart
>= 0)
1196 /* Reset everything */
1197 Timer
->DueTime
.QuadPart
= 0;
1199 Timer
->Header
.Hand
= 0;
1204 /* Update the interval */
1205 Interval
= TimeDifference
;
1209 /* Calculate the due time */
1210 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1211 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1212 Timer
->DueTime
.QuadPart
= DueTime
;
1214 /* Calculate the timer handle */
1215 *Hand
= KiComputeTimerTableIndex(DueTime
);
1216 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1219 #define KxDelayThreadWait() \
1221 /* Setup the Wait Block */ \
1222 Thread->WaitBlockList = TimerBlock; \
1224 /* Setup the timer */ \
1225 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1227 /* Save the due time for the caller */ \
1228 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1230 /* Link the timer to this Wait Block */ \
1231 TimerBlock->NextWaitBlock = TimerBlock; \
1232 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1233 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1235 /* Clear wait status */ \
1236 Thread->WaitStatus = STATUS_SUCCESS; \
1238 /* Setup wait fields */ \
1239 Thread->Alertable = Alertable; \
1240 Thread->WaitReason = DelayExecution; \
1241 Thread->WaitMode = WaitMode; \
1243 /* Check if we can swap the thread's stack */ \
1244 Thread->WaitListEntry.Flink = NULL; \
1245 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1247 /* Set the wait time */ \
1248 Thread->WaitTime = KeTickCount.LowPart;
1250 #define KxMultiThreadWait() \
1251 /* Link wait block array to the thread */ \
1252 Thread->WaitBlockList = WaitBlockArray; \
1254 /* Reset the index */ \
1257 /* Loop wait blocks */ \
1260 /* Fill out the wait block */ \
1261 WaitBlock = &WaitBlockArray[Index]; \
1262 WaitBlock->Object = Object[Index]; \
1263 WaitBlock->WaitKey = (USHORT)Index; \
1264 WaitBlock->WaitType = WaitType; \
1265 WaitBlock->Thread = Thread; \
1267 /* Link to next block */ \
1268 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1270 } while (Index < Count); \
1272 /* Link the last block */ \
1273 WaitBlock->NextWaitBlock = WaitBlockArray; \
1275 /* Set default wait status */ \
1276 Thread->WaitStatus = STATUS_WAIT_0; \
1278 /* Check if we have a timer */ \
1281 /* Link to the block */ \
1282 TimerBlock->NextWaitBlock = WaitBlockArray; \
1284 /* Setup the timer */ \
1285 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1287 /* Save the due time for the caller */ \
1288 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1290 /* Initialize the list */ \
1291 InitializeListHead(&Timer->Header.WaitListHead); \
1294 /* Set wait settings */ \
1295 Thread->Alertable = Alertable; \
1296 Thread->WaitMode = WaitMode; \
1297 Thread->WaitReason = WaitReason; \
1299 /* Check if we can swap the thread's stack */ \
1300 Thread->WaitListEntry.Flink = NULL; \
1301 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1303 /* Set the wait time */ \
1304 Thread->WaitTime = KeTickCount.LowPart;
1306 #define KxSingleThreadWait() \
1307 /* Setup the Wait Block */ \
1308 Thread->WaitBlockList = WaitBlock; \
1309 WaitBlock->WaitKey = STATUS_SUCCESS; \
1310 WaitBlock->Object = Object; \
1311 WaitBlock->WaitType = WaitAny; \
1313 /* Clear wait status */ \
1314 Thread->WaitStatus = STATUS_SUCCESS; \
1316 /* Check if we have a timer */ \
1319 /* Setup the timer */ \
1320 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1322 /* Save the due time for the caller */ \
1323 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1325 /* Pointer to timer block */ \
1326 WaitBlock->NextWaitBlock = TimerBlock; \
1327 TimerBlock->NextWaitBlock = WaitBlock; \
1329 /* Link the timer to this Wait Block */ \
1330 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1331 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1335 /* No timer block, just ourselves */ \
1336 WaitBlock->NextWaitBlock = WaitBlock; \
1339 /* Set wait settings */ \
1340 Thread->Alertable = Alertable; \
1341 Thread->WaitMode = WaitMode; \
1342 Thread->WaitReason = WaitReason; \
1344 /* Check if we can swap the thread's stack */ \
1345 Thread->WaitListEntry.Flink = NULL; \
1346 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1348 /* Set the wait time */ \
1349 Thread->WaitTime = KeTickCount.LowPart;
1351 #define KxQueueThreadWait() \
1352 /* Setup the Wait Block */ \
1353 Thread->WaitBlockList = WaitBlock; \
1354 WaitBlock->WaitKey = STATUS_SUCCESS; \
1355 WaitBlock->Object = Queue; \
1356 WaitBlock->WaitType = WaitAny; \
1357 WaitBlock->Thread = Thread; \
1359 /* Clear wait status */ \
1360 Thread->WaitStatus = STATUS_SUCCESS; \
1362 /* Check if we have a timer */ \
1365 /* Setup the timer */ \
1366 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1368 /* Save the due time for the caller */ \
1369 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1371 /* Pointer to timer block */ \
1372 WaitBlock->NextWaitBlock = TimerBlock; \
1373 TimerBlock->NextWaitBlock = WaitBlock; \
1375 /* Link the timer to this Wait Block */ \
1376 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1377 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1381 /* No timer block, just ourselves */ \
1382 WaitBlock->NextWaitBlock = WaitBlock; \
1385 /* Set wait settings */ \
1386 Thread->Alertable = FALSE; \
1387 Thread->WaitMode = WaitMode; \
1388 Thread->WaitReason = WrQueue; \
1390 /* Check if we can swap the thread's stack */ \
1391 Thread->WaitListEntry.Flink = NULL; \
1392 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1394 /* Set the wait time */ \
1395 Thread->WaitTime = KeTickCount.LowPart;
1402 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1403 IN KPRIORITY Increment
)
1405 PLIST_ENTRY WaitEntry
, WaitList
;
1406 PKWAIT_BLOCK WaitBlock
;
1407 PKTHREAD WaitThread
;
1410 /* Loop the Wait Entries */
1411 WaitList
= &Object
->WaitListHead
;
1412 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1413 WaitEntry
= WaitList
->Flink
;
1416 /* Get the current wait block */
1417 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1419 /* Get the waiting thread */
1420 WaitThread
= WaitBlock
->Thread
;
1422 /* Check the current Wait Mode */
1423 if (WaitBlock
->WaitType
== WaitAny
)
1425 /* Use the actual wait key */
1426 WaitKey
= WaitBlock
->WaitKey
;
1430 /* Otherwise, use STATUS_KERNEL_APC */
1431 WaitKey
= STATUS_KERNEL_APC
;
1434 /* Unwait the thread */
1435 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1438 WaitEntry
= WaitList
->Flink
;
1439 } while (WaitEntry
!= WaitList
);
1443 // Unwaits a Thread waiting on an event
1447 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1448 IN KPRIORITY Increment
)
1450 PLIST_ENTRY WaitEntry
, WaitList
;
1451 PKWAIT_BLOCK WaitBlock
;
1452 PKTHREAD WaitThread
;
1454 /* Loop the Wait Entries */
1455 WaitList
= &Event
->Header
.WaitListHead
;
1456 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1457 WaitEntry
= WaitList
->Flink
;
1460 /* Get the current wait block */
1461 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1463 /* Get the waiting thread */
1464 WaitThread
= WaitBlock
->Thread
;
1466 /* Check the current Wait Mode */
1467 if (WaitBlock
->WaitType
== WaitAny
)
1470 Event
->Header
.SignalState
= 0;
1472 /* Un-signal the event and unwait the thread */
1473 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1477 /* Unwait the thread with STATUS_KERNEL_APC */
1478 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1481 WaitEntry
= WaitList
->Flink
;
1482 } while (WaitEntry
!= WaitList
);
1486 // This routine queues a thread that is ready on the PRCB's ready lists.
1487 // If this thread cannot currently run on this CPU, then the thread is
1488 // added to the deferred ready list instead.
1490 // This routine must be entered with the PRCB lock held and it will exit
1491 // with the PRCB lock released!
1495 KxQueueReadyThread(IN PKTHREAD Thread
,
1502 ASSERT(Prcb
== KeGetCurrentPrcb());
1503 ASSERT(Thread
->State
== Running
);
1504 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1506 /* Check if this thread is allowed to run in this CPU */
1508 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1513 /* Set thread ready for execution */
1514 Thread
->State
= Ready
;
1516 /* Save current priority and if someone had pre-empted it */
1517 Priority
= Thread
->Priority
;
1518 Preempted
= Thread
->Preempted
;
1520 /* We're not pre-empting now, and set the wait time */
1521 Thread
->Preempted
= FALSE
;
1522 Thread
->WaitTime
= KeTickCount
.LowPart
;
1525 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1527 /* Insert this thread in the appropriate order */
1528 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1529 &Thread
->WaitListEntry
) :
1530 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1531 &Thread
->WaitListEntry
);
1533 /* Update the ready summary */
1534 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1537 ASSERT(Priority
== Thread
->Priority
);
1539 /* Release the PRCB lock */
1540 KiReleasePrcbLock(Prcb
);
1544 /* Otherwise, prepare this thread to be deferred */
1545 Thread
->State
= DeferredReady
;
1546 Thread
->DeferredProcessor
= Prcb
->Number
;
1548 /* Release the lock and defer scheduling */
1549 KiReleasePrcbLock(Prcb
);
1550 KiDeferredReadyThread(Thread
);
1555 // This routine scans for an appropriate ready thread to select at the
1556 // given priority and for the given CPU.
1560 KiSelectReadyThread(IN KPRIORITY Priority
,
1565 PLIST_ENTRY ListEntry
;
1566 PKTHREAD Thread
= NULL
;
1568 /* Save the current mask and get the priority set for the CPU */
1569 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1570 if (!PrioritySet
) goto Quickie
;
1572 /* Get the highest priority possible */
1573 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1574 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1575 HighPriority
+= Priority
;
1577 /* Make sure the list isn't empty at the highest priority */
1578 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1580 /* Get the first thread on the list */
1581 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1582 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1584 /* Make sure this thread is here for a reason */
1585 ASSERT(HighPriority
== Thread
->Priority
);
1586 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1587 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1589 /* Remove it from the list */
1590 if (RemoveEntryList(&Thread
->WaitListEntry
))
1592 /* The list is empty now, reset the ready summary */
1593 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1596 /* Sanity check and return the thread */
1598 ASSERT((Thread
== NULL
) ||
1599 (Thread
->BasePriority
== 0) ||
1600 (Thread
->Priority
!= 0));
1605 // This routine computes the new priority for a thread. It is only valid for
1606 // threads with priorities in the dynamic priority range.
1610 KiComputeNewPriority(IN PKTHREAD Thread
,
1611 IN SCHAR Adjustment
)
1615 /* Priority sanity checks */
1616 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1617 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1618 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1619 TRUE
: (Thread
->PriorityDecrement
== 0));
1621 /* Get the current priority */
1622 Priority
= Thread
->Priority
;
1623 if (Priority
< LOW_REALTIME_PRIORITY
)
1625 /* Decrease priority by the priority decrement */
1626 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1628 /* Don't go out of bounds */
1629 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1631 /* Reset the priority decrement */
1632 Thread
->PriorityDecrement
= 0;
1636 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1638 /* Return the new priority */
1643 // Guarded Mutex Routines
1647 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex
)
1649 /* Setup the Initial Data */
1650 GuardedMutex
->Count
= GM_LOCK_BIT
;
1651 GuardedMutex
->Owner
= NULL
;
1652 GuardedMutex
->Contention
= 0;
1654 /* Initialize the Wait Gate */
1655 KeInitializeGate(&GuardedMutex
->Gate
);
1660 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1662 PKTHREAD Thread
= KeGetCurrentThread();
1665 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1666 (Thread
->SpecialApcDisable
< 0) ||
1667 (Thread
->Teb
== NULL
) ||
1668 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1669 ASSERT(GuardedMutex
->Owner
!= Thread
);
1671 /* Remove the lock */
1672 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1674 /* The Guarded Mutex was already locked, enter contented case */
1675 KiAcquireGuardedMutex(GuardedMutex
);
1679 GuardedMutex
->Owner
= Thread
;
1684 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1686 LONG OldValue
, NewValue
;
1689 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1690 (KeGetCurrentThread()->SpecialApcDisable
< 0) ||
1691 (KeGetCurrentThread()->Teb
== NULL
) ||
1692 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1693 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1695 /* Destroy the Owner */
1696 GuardedMutex
->Owner
= NULL
;
1698 /* Add the Lock Bit */
1699 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1700 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1702 /* Check if it was already locked, but not woken */
1703 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1705 /* Update the Oldvalue to what it should be now */
1706 OldValue
+= GM_LOCK_BIT
;
1708 /* The mutex will be woken, minus one waiter */
1709 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1712 /* Remove the Woken bit */
1713 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1715 OldValue
) == OldValue
)
1717 /* Signal the Gate */
1718 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1725 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex
)
1727 PKTHREAD Thread
= KeGetCurrentThread();
1730 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1731 ASSERT(GuardedMutex
->Owner
!= Thread
);
1733 /* Disable Special APCs */
1734 KeEnterGuardedRegion();
1736 /* Remove the lock */
1737 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1739 /* The Guarded Mutex was already locked, enter contented case */
1740 KiAcquireGuardedMutex(GuardedMutex
);
1743 /* Set the Owner and Special APC Disable state */
1744 GuardedMutex
->Owner
= Thread
;
1745 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;
1750 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1752 LONG OldValue
, NewValue
;
1755 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1756 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1757 ASSERT(KeGetCurrentThread()->SpecialApcDisable
==
1758 GuardedMutex
->SpecialApcDisable
);
1760 /* Destroy the Owner */
1761 GuardedMutex
->Owner
= NULL
;
1763 /* Add the Lock Bit */
1764 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1765 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1767 /* Check if it was already locked, but not woken */
1768 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1770 /* Update the Oldvalue to what it should be now */
1771 OldValue
+= GM_LOCK_BIT
;
1773 /* The mutex will be woken, minus one waiter */
1774 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1777 /* Remove the Woken bit */
1778 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1780 OldValue
) == OldValue
)
1782 /* Signal the Gate */
1783 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1787 /* Re-enable APCs */
1788 KeLeaveGuardedRegion();
1793 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1795 PKTHREAD Thread
= KeGetCurrentThread();
1798 KeEnterGuardedRegion();
1800 /* Remove the lock */
1801 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1803 /* Re-enable APCs */
1804 KeLeaveGuardedRegion();
1807 /* Return failure */
1811 /* Set the Owner and APC State */
1812 GuardedMutex
->Owner
= Thread
;
1813 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;