2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
18 // Sanitizes a selector
22 Ke386SanitizeSeg(IN ULONG Cs
,
23 IN KPROCESSOR_MODE Mode
)
26 // Check if we're in kernel-mode, and force CPL 0 if so.
27 // Otherwise, force CPL 3.
29 return ((Mode
== KernelMode
) ?
30 (Cs
& (0xFFFF & ~RPL_MASK
)) :
31 (RPL_MASK
| (Cs
& 0xFFFF)));
39 Ke386SanitizeFlags(IN ULONG Eflags
,
40 IN KPROCESSOR_MODE Mode
)
43 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
44 // Otherwise, also force interrupt mask on.
46 return ((Mode
== KernelMode
) ?
47 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
48 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
52 // Gets a DR register from a CONTEXT structure
56 KiDrFromContext(IN ULONG Dr
,
59 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
63 // Gets a DR register from a KTRAP_FRAME structure
67 KiDrFromTrapFrame(IN ULONG Dr
,
68 IN PKTRAP_FRAME TrapFrame
)
70 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
78 Ke386SanitizeDr(IN PVOID DrAddress
,
79 IN KPROCESSOR_MODE Mode
)
82 // Check if we're in kernel-mode, and return the address directly if so.
83 // Otherwise, make sure it's not inside the kernel-mode address space.
84 // If it is, then clear the address.
86 return ((Mode
== KernelMode
) ? DrAddress
:
87 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
94 KeGetCurrentThread(VOID
)
97 /* Return the current thread */
98 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
99 #elif defined (_M_AMD64)
100 return (PRKTHREAD
)__readgsqword(FIELD_OFFSET(KIPCR
, Prcb
.CurrentThread
));
102 PKPRCB Prcb
= KeGetCurrentPrcb();
103 return Prcb
->CurrentThread
;
109 KeGetPreviousMode(VOID
)
111 /* Return the current mode */
112 return KeGetCurrentThread()->PreviousMode
;
118 KeFlushProcessTb(VOID
)
120 /* Flush the TLB by resetting CR3 */
122 __asm__("sync\n\tisync\n\t");
125 // We need to implement this!
127 ASSERTMSG("Need ARM flush routine\n", FALSE
);
129 __writecr3(__readcr3());
134 // Enters a Guarded Region
136 #define KeEnterGuardedRegion() \
138 PKTHREAD _Thread = KeGetCurrentThread(); \
140 /* Sanity checks */ \
141 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
142 ASSERT(_Thread == KeGetCurrentThread()); \
143 ASSERT((_Thread->SpecialApcDisable <= 0) && \
144 (_Thread->SpecialApcDisable != -32768)); \
146 /* Disable Special APCs */ \
147 _Thread->SpecialApcDisable--; \
151 // Leaves a Guarded Region
153 #define KeLeaveGuardedRegion() \
155 PKTHREAD _Thread = KeGetCurrentThread(); \
157 /* Sanity checks */ \
158 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->SpecialApcDisable < 0); \
162 /* Leave region and check if APCs are OK now */ \
163 if (!(++_Thread->SpecialApcDisable)) \
165 /* Check for Kernel APCs on the list */ \
166 if (!IsListEmpty(&_Thread->ApcState. \
167 ApcListHead[KernelMode])) \
169 /* Check for APC Delivery */ \
170 KiCheckForKernelApcDelivery(); \
176 // Enters a Critical Region
178 #define KeEnterCriticalRegion() \
180 PKTHREAD _Thread = KeGetCurrentThread(); \
182 /* Sanity checks */ \
183 ASSERT(_Thread == KeGetCurrentThread()); \
184 ASSERT((_Thread->KernelApcDisable <= 0) && \
185 (_Thread->KernelApcDisable != -32768)); \
187 /* Disable Kernel APCs */ \
188 _Thread->KernelApcDisable--; \
192 // Leaves a Critical Region
194 #define KeLeaveCriticalRegion() \
196 PKTHREAD _Thread = KeGetCurrentThread(); \
198 /* Sanity checks */ \
199 ASSERT(_Thread == KeGetCurrentThread()); \
200 ASSERT(_Thread->KernelApcDisable < 0); \
202 /* Enable Kernel APCs */ \
203 _Thread->KernelApcDisable++; \
205 /* Check if Kernel APCs are now enabled */ \
206 if (!(_Thread->KernelApcDisable)) \
208 /* Check if we need to request an APC Delivery */ \
209 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
210 !(_Thread->SpecialApcDisable)) \
212 /* Check for the right environment */ \
213 KiCheckForKernelApcDelivery(); \
220 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
224 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
226 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
227 UNREFERENCED_PARAMETER(SpinLock
);
231 // Spinlock Release at IRQL >= DISPATCH_LEVEL
235 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
237 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
238 UNREFERENCED_PARAMETER(SpinLock
);
242 // This routine protects against multiple CPU acquires, it's meaningless on UP.
246 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
248 UNREFERENCED_PARAMETER(Object
);
252 // This routine protects against multiple CPU acquires, it's meaningless on UP.
256 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
258 UNREFERENCED_PARAMETER(Object
);
263 KiAcquireDispatcherLock(VOID
)
265 /* Raise to DPC level */
266 return KeRaiseIrqlToDpcLevel();
271 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
273 /* Just exit the dispatcher */
274 KiExitDispatcher(OldIrql
);
279 KiAcquireDispatcherLockAtDpcLevel(VOID
)
281 /* This is a no-op at DPC Level for UP systems */
287 KiReleaseDispatcherLockFromDpcLevel(VOID
)
289 /* This is a no-op at DPC Level for UP systems */
294 // This routine makes the thread deferred ready on the boot CPU.
298 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
300 /* Set the thread to deferred state and boot CPU */
301 Thread
->State
= DeferredReady
;
302 Thread
->DeferredProcessor
= 0;
304 /* Make the thread ready immediately */
305 KiDeferredReadyThread(Thread
);
310 KiRescheduleThread(IN BOOLEAN NewThread
,
313 /* This is meaningless on UP systems */
314 UNREFERENCED_PARAMETER(NewThread
);
315 UNREFERENCED_PARAMETER(Cpu
);
319 // This routine protects against multiple CPU acquires, it's meaningless on UP.
323 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
325 UNREFERENCED_PARAMETER(Thread
);
329 // This routine protects against multiple CPU acquires, it's meaningless on UP.
333 KiAcquirePrcbLock(IN PKPRCB Prcb
)
335 UNREFERENCED_PARAMETER(Prcb
);
339 // This routine protects against multiple CPU acquires, it's meaningless on UP.
343 KiReleasePrcbLock(IN PKPRCB Prcb
)
345 UNREFERENCED_PARAMETER(Prcb
);
349 // This routine protects against multiple CPU acquires, it's meaningless on UP.
353 KiAcquireThreadLock(IN PKTHREAD Thread
)
355 UNREFERENCED_PARAMETER(Thread
);
359 // This routine protects against multiple CPU acquires, it's meaningless on UP.
363 KiReleaseThreadLock(IN PKTHREAD Thread
)
365 UNREFERENCED_PARAMETER(Thread
);
369 // This routine protects against multiple CPU acquires, it's meaningless on UP.
373 KiTryThreadLock(IN PKTHREAD Thread
)
375 UNREFERENCED_PARAMETER(Thread
);
381 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
383 /* There are no deferred ready lists on UP systems */
384 UNREFERENCED_PARAMETER(Prcb
);
389 KiRundownThread(IN PKTHREAD Thread
)
392 /* Check if this is the NPX Thread */
393 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
396 KeGetCurrentPrcb()->NpxThread
= NULL
;
404 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
407 /* We deliver instantly on UP */
408 UNREFERENCED_PARAMETER(NeedApc
);
409 UNREFERENCED_PARAMETER(Processor
);
414 KiAcquireTimerLock(IN ULONG Hand
)
416 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
418 /* Nothing to do on UP */
419 UNREFERENCED_PARAMETER(Hand
);
425 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
427 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
429 /* Nothing to do on UP */
430 UNREFERENCED_PARAMETER(LockQueue
);
436 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
440 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
442 /* Make sure that we don't own the lock already */
443 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) == *SpinLock
)
445 /* We do, bugcheck! */
446 KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
449 /* Start acquire loop */
452 /* Try to acquire it */
453 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
455 /* Value changed... wait until it's unlocked */
456 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
459 /* On debug builds, we use a much slower but useful routine */
460 //Kii386SpinOnSpinLock(SpinLock, 5);
462 /* FIXME: Do normal yield for now */
465 /* Otherwise, just yield and keep looping */
473 /* On debug builds, we OR in the KTHREAD */
474 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
476 /* All is well, break out */
483 // Spinlock Release at IRQL >= DISPATCH_LEVEL
487 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
490 /* Make sure that the threads match */
491 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
493 /* They don't, bugcheck */
494 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
498 InterlockedAnd((PLONG
)SpinLock
, 0);
503 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
507 /* Make sure we're at a safe level to touch the lock */
508 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
510 /* Start acquire loop */
513 /* Loop until the other CPU releases it */
516 /* Check if it got released */
517 OldValue
= Object
->Lock
;
518 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
520 /* Let the CPU know that this is a loop */
524 /* Try acquiring the lock now */
525 } while (InterlockedCompareExchange(&Object
->Lock
,
526 OldValue
| KOBJECT_LOCK_BIT
,
527 OldValue
) != OldValue
);
532 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
534 /* Make sure we're at a safe level to touch the lock */
535 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
538 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
543 KiAcquireDispatcherLock(VOID
)
545 /* Raise to synchronization level and acquire the dispatcher lock */
546 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
551 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
553 /* First release the lock */
554 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
555 LockQueue
[LockQueueDispatcherLock
]);
557 /* Then exit the dispatcher */
558 KiExitDispatcher(OldIrql
);
563 KiAcquireDispatcherLockAtDpcLevel(VOID
)
565 /* Acquire the dispatcher lock */
566 KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
567 LockQueue
[LockQueueDispatcherLock
]);
572 KiReleaseDispatcherLockFromDpcLevel(VOID
)
574 /* Release the dispatcher lock */
575 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
576 LockQueue
[LockQueueDispatcherLock
]);
580 // This routine inserts a thread into the deferred ready list of the current CPU
584 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
586 PKPRCB Prcb
= KeGetCurrentPrcb();
588 /* Set the thread to deferred state and CPU */
589 Thread
->State
= DeferredReady
;
590 Thread
->DeferredProcessor
= Prcb
->Number
;
592 /* Add it on the list */
593 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
598 KiRescheduleThread(IN BOOLEAN NewThread
,
601 /* Check if a new thread needs to be scheduled on a different CPU */
602 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
604 /* Send an IPI to request delivery */
605 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
610 // This routine sets the current thread in a swap busy state, which ensure that
611 // nobody else tries to swap it concurrently.
615 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
617 /* Make sure nobody already set it */
618 ASSERT(Thread
->SwapBusy
== FALSE
);
620 /* Set it ourselves */
621 Thread
->SwapBusy
= TRUE
;
625 // This routine acquires the PRCB lock so that only one caller can touch
626 // volatile PRCB data.
628 // Since this is a simple optimized spin-lock, it must only be acquired
629 // at dispatcher level or higher!
633 KiAcquirePrcbLock(IN PKPRCB Prcb
)
635 /* Make sure we're at a safe level to touch the PRCB lock */
636 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
638 /* Start acquire loop */
641 /* Acquire the lock and break out if we acquired it first */
642 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
644 /* Loop until the other CPU releases it */
647 /* Let the CPU know that this is a loop */
649 } while (Prcb
->PrcbLock
);
654 // This routine releases the PRCB lock so that other callers can touch
655 // volatile PRCB data.
657 // Since this is a simple optimized spin-lock, it must be be only acquired
658 // at dispatcher level or higher!
662 KiReleasePrcbLock(IN PKPRCB Prcb
)
664 /* Make sure we are above dispatch and the lock is acquired! */
665 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
666 ASSERT(Prcb
->PrcbLock
!= 0);
669 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
673 // This routine acquires the thread lock so that only one caller can touch
674 // volatile thread data.
676 // Since this is a simple optimized spin-lock, it must be be only acquired
677 // at dispatcher level or higher!
681 KiAcquireThreadLock(IN PKTHREAD Thread
)
683 /* Make sure we're at a safe level to touch the thread lock */
684 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
686 /* Start acquire loop */
689 /* Acquire the lock and break out if we acquired it first */
690 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
692 /* Loop until the other CPU releases it */
695 /* Let the CPU know that this is a loop */
697 } while (Thread
->ThreadLock
);
702 // This routine releases the thread lock so that other callers can touch
703 // volatile thread data.
705 // Since this is a simple optimized spin-lock, it must be be only acquired
706 // at dispatcher level or higher!
710 KiReleaseThreadLock(IN PKTHREAD Thread
)
712 /* Make sure we are still above dispatch */
713 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
716 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
721 KiTryThreadLock(IN PKTHREAD Thread
)
725 /* If the lock isn't acquired, return false */
726 if (!Thread
->ThreadLock
) return FALSE
;
728 /* Otherwise, try to acquire it and check the result */
730 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
732 /* Return the lock state */
733 return (Value
== TRUE
);
738 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
740 /* Scan the deferred ready lists if required */
741 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
746 KiRundownThread(IN PKTHREAD Thread
)
754 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
757 /* Check if we need to request APC delivery */
760 /* Check if it's on another CPU */
761 if (KeGetPcr()->Number
!= Processor
)
763 /* Send an IPI to request delivery */
764 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
768 /* Request a software interrupt */
769 HalRequestSoftwareInterrupt(APC_LEVEL
);
776 KiAcquireTimerLock(IN ULONG Hand
)
778 PKSPIN_LOCK_QUEUE LockQueue
;
780 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
782 /* Get the lock index */
783 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
784 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
786 /* Now get the lock */
787 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
789 /* Acquire it and return */
790 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
796 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
798 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
800 /* Release the lock */
801 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
808 KiAcquireApcLock(IN PKTHREAD Thread
,
809 IN PKLOCK_QUEUE_HANDLE Handle
)
811 /* Acquire the lock and raise to synchronization level */
812 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
817 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
818 IN PKLOCK_QUEUE_HANDLE Handle
)
820 /* Acquire the lock */
821 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
826 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
827 IN PKLOCK_QUEUE_HANDLE Handle
)
829 /* Acquire the lock */
830 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
835 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
837 /* Release the lock */
838 KeReleaseInStackQueuedSpinLock(Handle
);
843 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
845 /* Release the lock */
846 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
851 KiAcquireProcessLock(IN PKPROCESS Process
,
852 IN PKLOCK_QUEUE_HANDLE Handle
)
854 /* Acquire the lock and raise to synchronization level */
855 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
860 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
862 /* Release the lock */
863 KeReleaseInStackQueuedSpinLock(Handle
);
868 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
870 /* Release the lock */
871 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
876 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
877 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
879 /* Check if we were called from a threaded DPC */
880 if (KeGetCurrentPrcb()->DpcThreadActive
)
882 /* Lock the Queue, we're not at DPC level */
883 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
887 /* We must be at DPC level, acquire the lock safely */
888 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
889 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
896 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
898 /* Check if we were called from a threaded DPC */
899 if (KeGetCurrentPrcb()->DpcThreadActive
)
901 /* Unlock the Queue, we're not at DPC level */
902 KeReleaseInStackQueuedSpinLock(DeviceLock
);
906 /* We must be at DPC level, release the lock safely */
907 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
908 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
913 // Satisfies the wait of any dispatcher object
915 #define KiSatisfyObjectWait(Object, Thread) \
917 /* Special case for Mutants */ \
918 if ((Object)->Header.Type == MutantObject) \
920 /* Decrease the Signal State */ \
921 (Object)->Header.SignalState--; \
923 /* Check if it's now non-signaled */ \
924 if (!(Object)->Header.SignalState) \
926 /* Set the Owner Thread */ \
927 (Object)->OwnerThread = Thread; \
929 /* Disable APCs if needed */ \
930 Thread->KernelApcDisable = Thread->KernelApcDisable - \
931 (Object)->ApcDisable; \
933 /* Check if it's abandoned */ \
934 if ((Object)->Abandoned) \
937 (Object)->Abandoned = FALSE; \
939 /* Return Status */ \
940 Thread->WaitStatus = STATUS_ABANDONED; \
943 /* Insert it into the Mutant List */ \
944 InsertHeadList(Thread->MutantListHead.Blink, \
945 &(Object)->MutantListEntry); \
948 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
949 EventSynchronizationObject) \
951 /* Synchronization Timers and Events just get un-signaled */ \
952 (Object)->Header.SignalState = 0; \
954 else if ((Object)->Header.Type == SemaphoreObject) \
956 /* These ones can have multiple states, so we only decrease it */ \
957 (Object)->Header.SignalState--; \
962 // Satisfies the wait of a mutant dispatcher object
964 #define KiSatisfyMutantWait(Object, Thread) \
966 /* Decrease the Signal State */ \
967 (Object)->Header.SignalState--; \
969 /* Check if it's now non-signaled */ \
970 if (!(Object)->Header.SignalState) \
972 /* Set the Owner Thread */ \
973 (Object)->OwnerThread = Thread; \
975 /* Disable APCs if needed */ \
976 Thread->KernelApcDisable = Thread->KernelApcDisable - \
977 (Object)->ApcDisable; \
979 /* Check if it's abandoned */ \
980 if ((Object)->Abandoned) \
983 (Object)->Abandoned = FALSE; \
985 /* Return Status */ \
986 Thread->WaitStatus = STATUS_ABANDONED; \
989 /* Insert it into the Mutant List */ \
990 InsertHeadList(Thread->MutantListHead.Blink, \
991 &(Object)->MutantListEntry); \
996 // Satisfies the wait of any nonmutant dispatcher object
998 #define KiSatisfyNonMutantWait(Object) \
1000 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
1001 EventSynchronizationObject) \
1003 /* Synchronization Timers and Events just get un-signaled */ \
1004 (Object)->Header.SignalState = 0; \
1006 else if ((Object)->Header.Type == SemaphoreObject) \
1008 /* These ones can have multiple states, so we only decrease it */ \
1009 (Object)->Header.SignalState--; \
1014 // Recalculates the due time
1018 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
1019 IN PLARGE_INTEGER DueTime
,
1020 IN OUT PLARGE_INTEGER NewDueTime
)
1022 /* Don't do anything for absolute waits */
1023 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
1025 /* Otherwise, query the interrupt time and recalculate */
1026 NewDueTime
->QuadPart
= KeQueryInterruptTime();
1027 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
1032 // Determines whether a thread should be added to the wait list
1036 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
1037 IN KPROCESSOR_MODE WaitMode
)
1039 /* Check the required conditions */
1040 if ((WaitMode
!= KernelMode
) &&
1041 (Thread
->EnableStackSwap
) &&
1042 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
1044 /* We are go for swap */
1049 /* Don't swap the thread */
1055 // Adds a thread to the wait list
1057 #define KiAddThreadToWaitList(Thread, Swappable) \
1059 /* Make sure it's swappable */ \
1062 /* Insert it into the PRCB's List */ \
1063 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
1064 &Thread->WaitListEntry); \
1069 // Checks if a wait in progress should be interrupted by APCs or an alertable
1074 KiCheckAlertability(IN PKTHREAD Thread
,
1075 IN BOOLEAN Alertable
,
1076 IN KPROCESSOR_MODE WaitMode
)
1078 /* Check if the wait is alertable */
1081 /* It is, first check if the thread is alerted in this mode */
1082 if (Thread
->Alerted
[WaitMode
])
1084 /* It is, so bail out of the wait */
1085 Thread
->Alerted
[WaitMode
] = FALSE
;
1086 return STATUS_ALERTED
;
1088 else if ((WaitMode
!= KernelMode
) &&
1089 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
1091 /* It's isn't, but this is a user wait with queued user APCs */
1092 Thread
->ApcState
.UserApcPending
= TRUE
;
1093 return STATUS_USER_APC
;
1095 else if (Thread
->Alerted
[KernelMode
])
1097 /* It isn't that either, but we're alered in kernel mode */
1098 Thread
->Alerted
[KernelMode
] = FALSE
;
1099 return STATUS_ALERTED
;
1102 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
1104 /* Not alertable, but this is a user wait with pending user APCs */
1105 return STATUS_USER_APC
;
1108 /* Otherwise, we're fine */
1109 return STATUS_WAIT_0
;
1113 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
1114 // to remove timer entries
1115 // See Windows HPI blog for more information.
1118 KiRemoveEntryTimer(IN PKTIMER Timer
)
1121 PKTIMER_TABLE_ENTRY TableEntry
;
1123 /* Remove the timer from the timer list and check if it's empty */
1124 Hand
= Timer
->Header
.Hand
;
1125 if (RemoveEntryList(&Timer
->TimerListEntry
))
1127 /* Get the respective timer table entry */
1128 TableEntry
= &KiTimerTableListHead
[Hand
];
1129 if (&TableEntry
->Entry
== TableEntry
->Entry
.Flink
)
1131 /* Set the entry to an infinite absolute time */
1132 TableEntry
->Time
.HighPart
= 0xFFFFFFFF;
1136 /* Clear the list entries on dbg builds so we can tell the timer is gone */
1138 Timer
->TimerListEntry
.Flink
= NULL
;
1139 Timer
->TimerListEntry
.Blink
= NULL
;
1144 // Called by Wait and Queue code to insert a timer for dispatching.
1145 // Also called by KeSetTimerEx to insert a timer from the caller.
1149 KxInsertTimer(IN PKTIMER Timer
,
1152 PKSPIN_LOCK_QUEUE LockQueue
;
1154 /* Acquire the lock and release the dispatcher lock */
1155 LockQueue
= KiAcquireTimerLock(Hand
);
1156 KiReleaseDispatcherLockFromDpcLevel();
1158 /* Try to insert the timer */
1159 if (KiInsertTimerTable(Timer
, Hand
))
1162 KiCompleteTimer(Timer
, LockQueue
);
1166 /* Do nothing, just release the lock */
1167 KiReleaseTimerLock(LockQueue
);
1172 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
1173 // See the Windows HPI Blog for more information
1177 KiComputeDueTime(IN PKTIMER Timer
,
1178 IN LARGE_INTEGER DueTime
,
1181 LARGE_INTEGER InterruptTime
, SystemTime
, DifferenceTime
;
1183 /* Convert to relative time if needed */
1184 Timer
->Header
.Absolute
= FALSE
;
1185 if (DueTime
.HighPart
>= 0)
1187 /* Get System Time */
1188 KeQuerySystemTime(&SystemTime
);
1190 /* Do the conversion */
1191 DifferenceTime
.QuadPart
= SystemTime
.QuadPart
- DueTime
.QuadPart
;
1193 /* Make sure it hasn't already expired */
1194 Timer
->Header
.Absolute
= TRUE
;
1195 if (DifferenceTime
.HighPart
>= 0)
1197 /* Cancel everything */
1198 Timer
->Header
.SignalState
= TRUE
;
1199 Timer
->Header
.Hand
= 0;
1200 Timer
->DueTime
.QuadPart
= 0;
1205 /* Set the time as Absolute */
1206 DueTime
= DifferenceTime
;
1209 /* Get the Interrupt Time */
1210 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1212 /* Recalculate due time */
1213 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- DueTime
.QuadPart
;
1215 /* Get the handle */
1216 *Hand
= KiComputeTimerTableIndex(Timer
->DueTime
.QuadPart
);
1217 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1218 Timer
->Header
.Inserted
= TRUE
;
1223 // Called from Unlink and Queue Insert Code.
1224 // Also called by timer code when canceling an inserted timer.
1225 // Removes a timer from it's tree.
1229 KxRemoveTreeTimer(IN PKTIMER Timer
)
1231 ULONG Hand
= Timer
->Header
.Hand
;
1232 PKSPIN_LOCK_QUEUE LockQueue
;
1233 PKTIMER_TABLE_ENTRY TimerEntry
;
1235 /* Acquire timer lock */
1236 LockQueue
= KiAcquireTimerLock(Hand
);
1238 /* Set the timer as non-inserted */
1239 Timer
->Header
.Inserted
= FALSE
;
1241 /* Remove it from the timer list */
1242 if (RemoveEntryList(&Timer
->TimerListEntry
))
1244 /* Get the entry and check if it's empty */
1245 TimerEntry
= &KiTimerTableListHead
[Hand
];
1246 if (IsListEmpty(&TimerEntry
->Entry
))
1248 /* Clear the time then */
1249 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1253 /* Release the timer lock */
1254 KiReleaseTimerLock(LockQueue
);
1259 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1260 IN LARGE_INTEGER Interval
,
1264 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1266 /* Check the timer's interval to see if it's absolute */
1267 Timer
->Header
.Absolute
= FALSE
;
1268 if (Interval
.HighPart
>= 0)
1270 /* Get the system time and calculate the relative time */
1271 KeQuerySystemTime(&SystemTime
);
1272 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1273 Timer
->Header
.Absolute
= TRUE
;
1275 /* Check if we've already expired */
1276 if (TimeDifference
.HighPart
>= 0)
1278 /* Reset everything */
1279 Timer
->DueTime
.QuadPart
= 0;
1281 Timer
->Header
.Hand
= 0;
1286 /* Update the interval */
1287 Interval
= TimeDifference
;
1291 /* Calculate the due time */
1292 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1293 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1294 Timer
->DueTime
.QuadPart
= DueTime
;
1296 /* Calculate the timer handle */
1297 *Hand
= KiComputeTimerTableIndex(DueTime
);
1298 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1301 #define KxDelayThreadWait() \
1303 /* Setup the Wait Block */ \
1304 Thread->WaitBlockList = TimerBlock; \
1306 /* Setup the timer */ \
1307 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1309 /* Save the due time for the caller */ \
1310 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1312 /* Link the timer to this Wait Block */ \
1313 TimerBlock->NextWaitBlock = TimerBlock; \
1314 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1315 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1317 /* Clear wait status */ \
1318 Thread->WaitStatus = STATUS_SUCCESS; \
1320 /* Setup wait fields */ \
1321 Thread->Alertable = Alertable; \
1322 Thread->WaitReason = DelayExecution; \
1323 Thread->WaitMode = WaitMode; \
1325 /* Check if we can swap the thread's stack */ \
1326 Thread->WaitListEntry.Flink = NULL; \
1327 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1329 /* Set the wait time */ \
1330 Thread->WaitTime = KeTickCount.LowPart;
1332 #define KxMultiThreadWait() \
1333 /* Link wait block array to the thread */ \
1334 Thread->WaitBlockList = WaitBlockArray; \
1336 /* Reset the index */ \
1339 /* Loop wait blocks */ \
1342 /* Fill out the wait block */ \
1343 WaitBlock = &WaitBlockArray[Index]; \
1344 WaitBlock->Object = Object[Index]; \
1345 WaitBlock->WaitKey = (USHORT)Index; \
1346 WaitBlock->WaitType = WaitType; \
1347 WaitBlock->Thread = Thread; \
1349 /* Link to next block */ \
1350 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1352 } while (Index < Count); \
1354 /* Link the last block */ \
1355 WaitBlock->NextWaitBlock = WaitBlockArray; \
1357 /* Set default wait status */ \
1358 Thread->WaitStatus = STATUS_WAIT_0; \
1360 /* Check if we have a timer */ \
1363 /* Link to the block */ \
1364 TimerBlock->NextWaitBlock = WaitBlockArray; \
1366 /* Setup the timer */ \
1367 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1369 /* Save the due time for the caller */ \
1370 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1372 /* Initialize the list */ \
1373 InitializeListHead(&Timer->Header.WaitListHead); \
1376 /* Set wait settings */ \
1377 Thread->Alertable = Alertable; \
1378 Thread->WaitMode = WaitMode; \
1379 Thread->WaitReason = WaitReason; \
1381 /* Check if we can swap the thread's stack */ \
1382 Thread->WaitListEntry.Flink = NULL; \
1383 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1385 /* Set the wait time */ \
1386 Thread->WaitTime = KeTickCount.LowPart;
1388 #define KxSingleThreadWait() \
1389 /* Setup the Wait Block */ \
1390 Thread->WaitBlockList = WaitBlock; \
1391 WaitBlock->WaitKey = STATUS_SUCCESS; \
1392 WaitBlock->Object = Object; \
1393 WaitBlock->WaitType = WaitAny; \
1395 /* Clear wait status */ \
1396 Thread->WaitStatus = STATUS_SUCCESS; \
1398 /* Check if we have a timer */ \
1401 /* Setup the timer */ \
1402 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1404 /* Save the due time for the caller */ \
1405 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1407 /* Pointer to timer block */ \
1408 WaitBlock->NextWaitBlock = TimerBlock; \
1409 TimerBlock->NextWaitBlock = WaitBlock; \
1411 /* Link the timer to this Wait Block */ \
1412 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1413 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1417 /* No timer block, just ourselves */ \
1418 WaitBlock->NextWaitBlock = WaitBlock; \
1421 /* Set wait settings */ \
1422 Thread->Alertable = Alertable; \
1423 Thread->WaitMode = WaitMode; \
1424 Thread->WaitReason = WaitReason; \
1426 /* Check if we can swap the thread's stack */ \
1427 Thread->WaitListEntry.Flink = NULL; \
1428 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1430 /* Set the wait time */ \
1431 Thread->WaitTime = KeTickCount.LowPart;
1433 #define KxQueueThreadWait() \
1434 /* Setup the Wait Block */ \
1435 Thread->WaitBlockList = WaitBlock; \
1436 WaitBlock->WaitKey = STATUS_SUCCESS; \
1437 WaitBlock->Object = Queue; \
1438 WaitBlock->WaitType = WaitAny; \
1439 WaitBlock->Thread = Thread; \
1441 /* Clear wait status */ \
1442 Thread->WaitStatus = STATUS_SUCCESS; \
1444 /* Check if we have a timer */ \
1447 /* Setup the timer */ \
1448 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1450 /* Save the due time for the caller */ \
1451 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1453 /* Pointer to timer block */ \
1454 WaitBlock->NextWaitBlock = TimerBlock; \
1455 TimerBlock->NextWaitBlock = WaitBlock; \
1457 /* Link the timer to this Wait Block */ \
1458 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1459 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1463 /* No timer block, just ourselves */ \
1464 WaitBlock->NextWaitBlock = WaitBlock; \
1467 /* Set wait settings */ \
1468 Thread->Alertable = FALSE; \
1469 Thread->WaitMode = WaitMode; \
1470 Thread->WaitReason = WrQueue; \
1472 /* Check if we can swap the thread's stack */ \
1473 Thread->WaitListEntry.Flink = NULL; \
1474 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1476 /* Set the wait time */ \
1477 Thread->WaitTime = KeTickCount.LowPart;
1484 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1485 IN KPRIORITY Increment
)
1487 PLIST_ENTRY WaitEntry
, WaitList
;
1488 PKWAIT_BLOCK WaitBlock
;
1489 PKTHREAD WaitThread
;
1492 /* Loop the Wait Entries */
1493 WaitList
= &Object
->WaitListHead
;
1494 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1495 WaitEntry
= WaitList
->Flink
;
1498 /* Get the current wait block */
1499 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1501 /* Get the waiting thread */
1502 WaitThread
= WaitBlock
->Thread
;
1504 /* Check the current Wait Mode */
1505 if (WaitBlock
->WaitType
== WaitAny
)
1507 /* Use the actual wait key */
1508 WaitKey
= WaitBlock
->WaitKey
;
1512 /* Otherwise, use STATUS_KERNEL_APC */
1513 WaitKey
= STATUS_KERNEL_APC
;
1516 /* Unwait the thread */
1517 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1520 WaitEntry
= WaitList
->Flink
;
1521 } while (WaitEntry
!= WaitList
);
1525 // Unwaits a Thread waiting on an event
1529 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1530 IN KPRIORITY Increment
)
1532 PLIST_ENTRY WaitEntry
, WaitList
;
1533 PKWAIT_BLOCK WaitBlock
;
1534 PKTHREAD WaitThread
;
1536 /* Loop the Wait Entries */
1537 WaitList
= &Event
->Header
.WaitListHead
;
1538 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1539 WaitEntry
= WaitList
->Flink
;
1542 /* Get the current wait block */
1543 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1545 /* Get the waiting thread */
1546 WaitThread
= WaitBlock
->Thread
;
1548 /* Check the current Wait Mode */
1549 if (WaitBlock
->WaitType
== WaitAny
)
1552 Event
->Header
.SignalState
= 0;
1554 /* Un-signal the event and unwait the thread */
1555 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1559 /* Unwait the thread with STATUS_KERNEL_APC */
1560 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1563 WaitEntry
= WaitList
->Flink
;
1564 } while (WaitEntry
!= WaitList
);
1568 // This routine queues a thread that is ready on the PRCB's ready lists.
1569 // If this thread cannot currently run on this CPU, then the thread is
1570 // added to the deferred ready list instead.
1572 // This routine must be entered with the PRCB lock held and it will exit
1573 // with the PRCB lock released!
1577 KxQueueReadyThread(IN PKTHREAD Thread
,
1584 ASSERT(Prcb
== KeGetCurrentPrcb());
1585 ASSERT(Thread
->State
== Running
);
1586 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1588 /* Check if this thread is allowed to run in this CPU */
1590 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1595 /* Set thread ready for execution */
1596 Thread
->State
= Ready
;
1598 /* Save current priority and if someone had pre-empted it */
1599 Priority
= Thread
->Priority
;
1600 Preempted
= Thread
->Preempted
;
1602 /* We're not pre-empting now, and set the wait time */
1603 Thread
->Preempted
= FALSE
;
1604 Thread
->WaitTime
= KeTickCount
.LowPart
;
1607 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1609 /* Insert this thread in the appropriate order */
1610 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1611 &Thread
->WaitListEntry
) :
1612 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1613 &Thread
->WaitListEntry
);
1615 /* Update the ready summary */
1616 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1619 ASSERT(Priority
== Thread
->Priority
);
1621 /* Release the PRCB lock */
1622 KiReleasePrcbLock(Prcb
);
1626 /* Otherwise, prepare this thread to be deferred */
1627 Thread
->State
= DeferredReady
;
1628 Thread
->DeferredProcessor
= Prcb
->Number
;
1630 /* Release the lock and defer scheduling */
1631 KiReleasePrcbLock(Prcb
);
1632 KiDeferredReadyThread(Thread
);
1637 // This routine scans for an appropriate ready thread to select at the
1638 // given priority and for the given CPU.
1642 KiSelectReadyThread(IN KPRIORITY Priority
,
1647 PLIST_ENTRY ListEntry
;
1648 PKTHREAD Thread
= NULL
;
1650 /* Save the current mask and get the priority set for the CPU */
1651 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1652 if (!PrioritySet
) goto Quickie
;
1654 /* Get the highest priority possible */
1655 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1656 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1657 HighPriority
+= Priority
;
1659 /* Make sure the list isn't empty at the highest priority */
1660 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1662 /* Get the first thread on the list */
1663 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1664 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1666 /* Make sure this thread is here for a reason */
1667 ASSERT(HighPriority
== Thread
->Priority
);
1668 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1669 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1671 /* Remove it from the list */
1672 if (RemoveEntryList(&Thread
->WaitListEntry
))
1674 /* The list is empty now, reset the ready summary */
1675 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1678 /* Sanity check and return the thread */
1680 ASSERT((Thread
== NULL
) ||
1681 (Thread
->BasePriority
== 0) ||
1682 (Thread
->Priority
!= 0));
1687 // This routine computes the new priority for a thread. It is only valid for
1688 // threads with priorities in the dynamic priority range.
1692 KiComputeNewPriority(IN PKTHREAD Thread
,
1693 IN SCHAR Adjustment
)
1697 /* Priority sanity checks */
1698 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1699 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1700 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1701 TRUE
: (Thread
->PriorityDecrement
== 0));
1703 /* Get the current priority */
1704 Priority
= Thread
->Priority
;
1705 if (Priority
< LOW_REALTIME_PRIORITY
)
1707 /* Decrease priority by the priority decrement */
1708 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1710 /* Don't go out of bounds */
1711 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1713 /* Reset the priority decrement */
1714 Thread
->PriorityDecrement
= 0;
1718 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1720 /* Return the new priority */
1725 // Guarded Mutex Routines
1729 _KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex
)
1731 /* Setup the Initial Data */
1732 GuardedMutex
->Count
= GM_LOCK_BIT
;
1733 GuardedMutex
->Owner
= NULL
;
1734 GuardedMutex
->Contention
= 0;
1736 /* Initialize the Wait Gate */
1737 KeInitializeGate(&GuardedMutex
->Gate
);
1742 _KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1744 PKTHREAD Thread
= KeGetCurrentThread();
1747 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1748 (Thread
->SpecialApcDisable
< 0) ||
1749 (Thread
->Teb
== NULL
) ||
1750 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1751 ASSERT(GuardedMutex
->Owner
!= Thread
);
1753 /* Remove the lock */
1754 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1756 /* The Guarded Mutex was already locked, enter contented case */
1757 KiAcquireGuardedMutex(GuardedMutex
);
1761 GuardedMutex
->Owner
= Thread
;
1766 _KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1768 LONG OldValue
, NewValue
;
1771 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1772 (KeGetCurrentThread()->SpecialApcDisable
< 0) ||
1773 (KeGetCurrentThread()->Teb
== NULL
) ||
1774 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1775 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1777 /* Destroy the Owner */
1778 GuardedMutex
->Owner
= NULL
;
1780 /* Add the Lock Bit */
1781 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1782 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1784 /* Check if it was already locked, but not woken */
1785 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1787 /* Update the Oldvalue to what it should be now */
1788 OldValue
+= GM_LOCK_BIT
;
1790 /* The mutex will be woken, minus one waiter */
1791 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1794 /* Remove the Woken bit */
1795 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1797 OldValue
) == OldValue
)
1799 /* Signal the Gate */
1800 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1807 _KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex
)
1809 PKTHREAD Thread
= KeGetCurrentThread();
1812 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1813 ASSERT(GuardedMutex
->Owner
!= Thread
);
1815 /* Disable Special APCs */
1816 KeEnterGuardedRegion();
1818 /* Remove the lock */
1819 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1821 /* The Guarded Mutex was already locked, enter contented case */
1822 KiAcquireGuardedMutex(GuardedMutex
);
1825 /* Set the Owner and Special APC Disable state */
1826 GuardedMutex
->Owner
= Thread
;
1827 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;
1832 _KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1834 LONG OldValue
, NewValue
;
1837 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1838 ASSERT(GuardedMutex
->Owner
== KeGetCurrentThread());
1839 ASSERT(KeGetCurrentThread()->SpecialApcDisable
==
1840 GuardedMutex
->SpecialApcDisable
);
1842 /* Destroy the Owner */
1843 GuardedMutex
->Owner
= NULL
;
1845 /* Add the Lock Bit */
1846 OldValue
= InterlockedExchangeAdd(&GuardedMutex
->Count
, GM_LOCK_BIT
);
1847 ASSERT((OldValue
& GM_LOCK_BIT
) == 0);
1849 /* Check if it was already locked, but not woken */
1850 if ((OldValue
) && !(OldValue
& GM_LOCK_WAITER_WOKEN
))
1852 /* Update the Oldvalue to what it should be now */
1853 OldValue
+= GM_LOCK_BIT
;
1855 /* The mutex will be woken, minus one waiter */
1856 NewValue
= OldValue
+ GM_LOCK_WAITER_WOKEN
-
1859 /* Remove the Woken bit */
1860 if (InterlockedCompareExchange(&GuardedMutex
->Count
,
1862 OldValue
) == OldValue
)
1864 /* Signal the Gate */
1865 KeSignalGateBoostPriority(&GuardedMutex
->Gate
);
1869 /* Re-enable APCs */
1870 KeLeaveGuardedRegion();
1875 _KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex
)
1877 PKTHREAD Thread
= KeGetCurrentThread();
1880 KeEnterGuardedRegion();
1882 /* Remove the lock */
1883 if (!InterlockedBitTestAndReset(&GuardedMutex
->Count
, GM_LOCK_BIT_V
))
1885 /* Re-enable APCs */
1886 KeLeaveGuardedRegion();
1889 /* Return failure */
1893 /* Set the Owner and APC State */
1894 GuardedMutex
->Owner
= Thread
;
1895 GuardedMutex
->SpecialApcDisable
= Thread
->SpecialApcDisable
;