2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
18 // Sanitizes a selector
22 Ke386SanitizeSeg(IN ULONG Cs
,
23 IN KPROCESSOR_MODE Mode
)
26 // Check if we're in kernel-mode, and force CPL 0 if so.
27 // Otherwise, force CPL 3.
29 return ((Mode
== KernelMode
) ?
30 (Cs
& (0xFFFF & ~RPL_MASK
)) :
31 (RPL_MASK
| (Cs
& 0xFFFF)));
39 Ke386SanitizeFlags(IN ULONG Eflags
,
40 IN KPROCESSOR_MODE Mode
)
43 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
44 // Otherwise, also force interrupt mask on.
46 return ((Mode
== KernelMode
) ?
47 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
48 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
52 // Gets a DR register from a CONTEXT structure
56 KiDrFromContext(IN ULONG Dr
,
59 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
63 // Gets a DR register from a KTRAP_FRAME structure
67 KiDrFromTrapFrame(IN ULONG Dr
,
68 IN PKTRAP_FRAME TrapFrame
)
70 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
78 Ke386SanitizeDr(IN PVOID DrAddress
,
79 IN KPROCESSOR_MODE Mode
)
82 // Check if we're in kernel-mode, and return the address directly if so.
83 // Otherwise, make sure it's not inside the kernel-mode address space.
84 // If it is, then clear the address.
86 return ((Mode
== KernelMode
) ? DrAddress
:
87 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
94 KeGetCurrentThread(VOID
)
97 /* Return the current thread */
98 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
100 PKPRCB Prcb
= KeGetCurrentPrcb();
101 return Prcb
->CurrentThread
;
107 KeGetPreviousMode(VOID
)
109 /* Return the current mode */
110 return KeGetCurrentThread()->PreviousMode
;
116 KeFlushProcessTb(VOID
)
118 /* Flush the TLB by resetting CR3 */
120 __asm__("sync\n\tisync\n\t");
123 // We need to implement this!
125 ASSERTMSG("Need ARM flush routine\n", FALSE
);
127 __writecr3(__readcr3());
132 // Enters a Guarded Region
134 #define KeEnterGuardedRegion() \
136 PKTHREAD _Thread = KeGetCurrentThread(); \
138 /* Sanity checks */ \
139 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
140 ASSERT(_Thread == KeGetCurrentThread()); \
141 ASSERT((_Thread->SpecialApcDisable <= 0) && \
142 (_Thread->SpecialApcDisable != -32768)); \
144 /* Disable Special APCs */ \
145 _Thread->SpecialApcDisable--; \
149 // Leaves a Guarded Region
151 #define KeLeaveGuardedRegion() \
153 PKTHREAD _Thread = KeGetCurrentThread(); \
155 /* Sanity checks */ \
156 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
157 ASSERT(_Thread == KeGetCurrentThread()); \
158 ASSERT(_Thread->SpecialApcDisable < 0); \
160 /* Leave region and check if APCs are OK now */ \
161 if (!(++_Thread->SpecialApcDisable)) \
163 /* Check for Kernel APCs on the list */ \
164 if (!IsListEmpty(&_Thread->ApcState. \
165 ApcListHead[KernelMode])) \
167 /* Check for APC Delivery */ \
168 KiCheckForKernelApcDelivery(); \
174 // TODO: Guarded Mutex Routines
178 // Enters a Critical Region
180 #define KeEnterCriticalRegion() \
182 PKTHREAD _Thread = KeGetCurrentThread(); \
184 /* Sanity checks */ \
185 ASSERT(_Thread == KeGetCurrentThread()); \
186 ASSERT((_Thread->KernelApcDisable <= 0) && \
187 (_Thread->KernelApcDisable != -32768)); \
189 /* Disable Kernel APCs */ \
190 _Thread->KernelApcDisable--; \
194 // Leaves a Critical Region
196 #define KeLeaveCriticalRegion() \
198 PKTHREAD _Thread = KeGetCurrentThread(); \
200 /* Sanity checks */ \
201 ASSERT(_Thread == KeGetCurrentThread()); \
202 ASSERT(_Thread->KernelApcDisable < 0); \
204 /* Enable Kernel APCs */ \
205 _Thread->KernelApcDisable++; \
207 /* Check if Kernel APCs are now enabled */ \
208 if (!(_Thread->KernelApcDisable)) \
210 /* Check if we need to request an APC Delivery */ \
211 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
212 !(_Thread->SpecialApcDisable)) \
214 /* Check for the right environment */ \
215 KiCheckForKernelApcDelivery(); \
222 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
226 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
228 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
229 UNREFERENCED_PARAMETER(SpinLock
);
233 // Spinlock Release at IRQL >= DISPATCH_LEVEL
237 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
239 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
240 UNREFERENCED_PARAMETER(SpinLock
);
244 // This routine protects against multiple CPU acquires, it's meaningless on UP.
248 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
250 UNREFERENCED_PARAMETER(Object
);
254 // This routine protects against multiple CPU acquires, it's meaningless on UP.
258 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
260 UNREFERENCED_PARAMETER(Object
);
265 KiAcquireDispatcherLock(VOID
)
267 /* Raise to DPC level */
268 return KeRaiseIrqlToDpcLevel();
273 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
275 /* Just exit the dispatcher */
276 KiExitDispatcher(OldIrql
);
281 KiAcquireDispatcherLockAtDpcLevel(VOID
)
283 /* This is a no-op at DPC Level for UP systems */
289 KiReleaseDispatcherLockFromDpcLevel(VOID
)
291 /* This is a no-op at DPC Level for UP systems */
296 // This routine makes the thread deferred ready on the boot CPU.
300 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
302 /* Set the thread to deferred state and boot CPU */
303 Thread
->State
= DeferredReady
;
304 Thread
->DeferredProcessor
= 0;
306 /* Make the thread ready immediately */
307 KiDeferredReadyThread(Thread
);
312 KiRescheduleThread(IN BOOLEAN NewThread
,
315 /* This is meaningless on UP systems */
316 UNREFERENCED_PARAMETER(NewThread
);
317 UNREFERENCED_PARAMETER(Cpu
);
321 // This routine protects against multiple CPU acquires, it's meaningless on UP.
325 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
327 UNREFERENCED_PARAMETER(Thread
);
331 // This routine protects against multiple CPU acquires, it's meaningless on UP.
335 KiAcquirePrcbLock(IN PKPRCB Prcb
)
337 UNREFERENCED_PARAMETER(Prcb
);
341 // This routine protects against multiple CPU acquires, it's meaningless on UP.
345 KiReleasePrcbLock(IN PKPRCB Prcb
)
347 UNREFERENCED_PARAMETER(Prcb
);
351 // This routine protects against multiple CPU acquires, it's meaningless on UP.
355 KiAcquireThreadLock(IN PKTHREAD Thread
)
357 UNREFERENCED_PARAMETER(Thread
);
361 // This routine protects against multiple CPU acquires, it's meaningless on UP.
365 KiReleaseThreadLock(IN PKTHREAD Thread
)
367 UNREFERENCED_PARAMETER(Thread
);
371 // This routine protects against multiple CPU acquires, it's meaningless on UP.
375 KiTryThreadLock(IN PKTHREAD Thread
)
377 UNREFERENCED_PARAMETER(Thread
);
383 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
385 /* There are no deferred ready lists on UP systems */
386 UNREFERENCED_PARAMETER(Prcb
);
391 KiRundownThread(IN PKTHREAD Thread
)
393 #if defined(_M_IX86) || defined(_M_AMD64)
394 /* Check if this is the NPX Thread */
395 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
398 KeGetCurrentPrcb()->NpxThread
= NULL
;
406 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
409 /* We deliver instantly on UP */
410 UNREFERENCED_PARAMETER(NeedApc
);
411 UNREFERENCED_PARAMETER(Processor
);
416 KiAcquireTimerLock(IN ULONG Hand
)
418 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
420 /* Nothing to do on UP */
421 UNREFERENCED_PARAMETER(Hand
);
427 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
429 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
431 /* Nothing to do on UP */
432 UNREFERENCED_PARAMETER(LockQueue
);
438 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
442 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
446 /* Try to acquire it */
447 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
449 /* Value changed... wait until it's locked */
450 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
453 /* On debug builds, we use a much slower but useful routine */
454 //Kii386SpinOnSpinLock(SpinLock, 5);
456 /* FIXME: Do normal yield for now */
459 /* Otherwise, just yield and keep looping */
467 /* On debug builds, we OR in the KTHREAD */
468 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
470 /* All is well, break out */
477 // Spinlock Release at IRQL >= DISPATCH_LEVEL
481 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
484 /* Make sure that the threads match */
485 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
487 /* They don't, bugcheck */
488 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
492 InterlockedAnd((PLONG
)SpinLock
, 0);
497 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
501 /* Make sure we're at a safe level to touch the lock */
502 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
504 /* Start acquire loop */
507 /* Loop until the other CPU releases it */
510 /* Check if it got released */
511 OldValue
= Object
->Lock
;
512 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
514 /* Let the CPU know that this is a loop */
518 /* Try acquiring the lock now */
519 } while (InterlockedCompareExchange(&Object
->Lock
,
520 OldValue
| KOBJECT_LOCK_BIT
,
521 OldValue
) != OldValue
);
526 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
528 /* Make sure we're at a safe level to touch the lock */
529 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
532 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
537 KiAcquireDispatcherLock(VOID
)
539 /* Raise to synchronization level and acquire the dispatcher lock */
540 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
545 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
547 /* First release the lock */
548 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
549 LockQueue
[LockQueueDispatcherLock
]);
551 /* Then exit the dispatcher */
552 KiExitDispatcher(OldIrql
);
557 KiAcquireDispatcherLockAtDpcLevel(VOID
)
559 /* Acquire the dispatcher lock */
560 KeAcquireQueuedSpinLockAtDpcLevel(LockQueueDispatcherLock
);
565 KiReleaseDispatcherLockFromDpcLevel(VOID
)
567 /* Release the dispatcher lock */
568 KeReleaseQueuedSpinLockFromDpcLevel(LockQueueDispatcherLock
);
572 // This routine inserts a thread into the deferred ready list of the given CPU
576 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
578 PKPRCB Prcb
= KeGetCurrentPrcb();
580 /* Set the thread to deferred state and CPU */
581 Thread
->State
= DeferredReady
;
582 Thread
->DeferredProcessor
= Prcb
->Number
;
584 /* Add it on the list */
585 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
590 KiRescheduleThread(IN BOOLEAN NewThread
,
593 /* Check if a new thread needs to be scheduled on a different CPU */
594 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
596 /* Send an IPI to request delivery */
597 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
602 // This routine sets the current thread in a swap busy state, which ensure that
603 // nobody else tries to swap it concurrently.
607 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
609 /* Make sure nobody already set it */
610 ASSERT(Thread
->SwapBusy
== FALSE
);
612 /* Set it ourselves */
613 Thread
->SwapBusy
= TRUE
;
617 // This routine acquires the PRCB lock so that only one caller can touch
618 // volatile PRCB data.
620 // Since this is a simple optimized spin-lock, it must be be only acquired
621 // at dispatcher level or higher!
625 KiAcquirePrcbLock(IN PKPRCB Prcb
)
627 /* Make sure we're at a safe level to touch the PRCB lock */
628 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
630 /* Start acquire loop */
633 /* Acquire the lock and break out if we acquired it first */
634 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
636 /* Loop until the other CPU releases it */
639 /* Let the CPU know that this is a loop */
641 } while (Prcb
->PrcbLock
);
646 // This routine releases the PRCB lock so that other callers can touch
647 // volatile PRCB data.
649 // Since this is a simple optimized spin-lock, it must be be only acquired
650 // at dispatcher level or higher!
654 KiReleasePrcbLock(IN PKPRCB Prcb
)
656 /* Make sure it's acquired! */
657 ASSERT(Prcb
->PrcbLock
!= 0);
660 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
664 // This routine acquires the thread lock so that only one caller can touch
665 // volatile thread data.
667 // Since this is a simple optimized spin-lock, it must be be only acquired
668 // at dispatcher level or higher!
672 KiAcquireThreadLock(IN PKTHREAD Thread
)
674 /* Make sure we're at a safe level to touch the thread lock */
675 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
677 /* Start acquire loop */
680 /* Acquire the lock and break out if we acquired it first */
681 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
683 /* Loop until the other CPU releases it */
686 /* Let the CPU know that this is a loop */
688 } while (Thread
->ThreadLock
);
693 // This routine releases the thread lock so that other callers can touch
694 // volatile thread data.
696 // Since this is a simple optimized spin-lock, it must be be only acquired
697 // at dispatcher level or higher!
701 KiReleaseThreadLock(IN PKTHREAD Thread
)
704 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
709 KiTryThreadLock(IN PKTHREAD Thread
)
713 /* If the lock isn't acquired, return false */
714 if (!Thread
->ThreadLock
) return FALSE
;
716 /* Otherwise, try to acquire it and check the result */
718 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
720 /* Return the lock state */
721 return (Value
== TRUE
);
726 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
728 /* Scan the deferred ready lists if required */
729 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
734 KiRundownThread(IN PKTHREAD Thread
)
736 #if defined(_M_IX86) || defined(_M_AMD64)
738 ASSERTMSG("Not yet implemented\n", FALSE
);
744 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
747 /* Check if we need to request APC delivery */
750 /* Check if it's on another CPU */
751 if (KeGetPcr()->Number
!= Processor
)
753 /* Send an IPI to request delivery */
754 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
758 /* Request a software interrupt */
759 HalRequestSoftwareInterrupt(APC_LEVEL
);
766 KiAcquireTimerLock(IN ULONG Hand
)
768 PKSPIN_LOCK_QUEUE LockQueue
;
770 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
772 /* Get the lock index */
773 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
774 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
776 /* Now get the lock */
777 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
779 /* Acquire it and return */
780 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
786 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
788 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
790 /* Release the lock */
791 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
798 KiAcquireApcLock(IN PKTHREAD Thread
,
799 IN PKLOCK_QUEUE_HANDLE Handle
)
801 /* Acquire the lock and raise to synchronization level */
802 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
807 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
808 IN PKLOCK_QUEUE_HANDLE Handle
)
810 /* Acquire the lock */
811 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
816 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
817 IN PKLOCK_QUEUE_HANDLE Handle
)
819 /* Acquire the lock */
820 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
825 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
827 /* Release the lock */
828 KeReleaseInStackQueuedSpinLock(Handle
);
833 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
835 /* Release the lock */
836 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
841 KiAcquireProcessLock(IN PKPROCESS Process
,
842 IN PKLOCK_QUEUE_HANDLE Handle
)
844 /* Acquire the lock and raise to synchronization level */
845 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
850 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
852 /* Release the lock */
853 KeReleaseInStackQueuedSpinLock(Handle
);
858 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
860 /* Release the lock */
861 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
866 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
867 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
869 /* Check if we were called from a threaded DPC */
870 if (KeGetCurrentPrcb()->DpcThreadActive
)
872 /* Lock the Queue, we're not at DPC level */
873 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
877 /* We must be at DPC level, acquire the lock safely */
878 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
879 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
886 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
888 /* Check if we were called from a threaded DPC */
889 if (KeGetCurrentPrcb()->DpcThreadActive
)
891 /* Unlock the Queue, we're not at DPC level */
892 KeReleaseInStackQueuedSpinLock(DeviceLock
);
896 /* We must be at DPC level, release the lock safely */
897 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
898 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
903 // Satisfies the wait of any dispatcher object
905 #define KiSatisfyObjectWait(Object, Thread) \
907 /* Special case for Mutants */ \
908 if ((Object)->Header.Type == MutantObject) \
910 /* Decrease the Signal State */ \
911 (Object)->Header.SignalState--; \
913 /* Check if it's now non-signaled */ \
914 if (!(Object)->Header.SignalState) \
916 /* Set the Owner Thread */ \
917 (Object)->OwnerThread = Thread; \
919 /* Disable APCs if needed */ \
920 Thread->KernelApcDisable = Thread->KernelApcDisable - \
921 (Object)->ApcDisable; \
923 /* Check if it's abandoned */ \
924 if ((Object)->Abandoned) \
927 (Object)->Abandoned = FALSE; \
929 /* Return Status */ \
930 Thread->WaitStatus = STATUS_ABANDONED; \
933 /* Insert it into the Mutant List */ \
934 InsertHeadList(Thread->MutantListHead.Blink, \
935 &(Object)->MutantListEntry); \
938 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
939 EventSynchronizationObject) \
941 /* Synchronization Timers and Events just get un-signaled */ \
942 (Object)->Header.SignalState = 0; \
944 else if ((Object)->Header.Type == SemaphoreObject) \
946 /* These ones can have multiple states, so we only decrease it */ \
947 (Object)->Header.SignalState--; \
952 // Satisfies the wait of a mutant dispatcher object
954 #define KiSatisfyMutantWait(Object, Thread) \
956 /* Decrease the Signal State */ \
957 (Object)->Header.SignalState--; \
959 /* Check if it's now non-signaled */ \
960 if (!(Object)->Header.SignalState) \
962 /* Set the Owner Thread */ \
963 (Object)->OwnerThread = Thread; \
965 /* Disable APCs if needed */ \
966 Thread->KernelApcDisable = Thread->KernelApcDisable - \
967 (Object)->ApcDisable; \
969 /* Check if it's abandoned */ \
970 if ((Object)->Abandoned) \
973 (Object)->Abandoned = FALSE; \
975 /* Return Status */ \
976 Thread->WaitStatus = STATUS_ABANDONED; \
979 /* Insert it into the Mutant List */ \
980 InsertHeadList(Thread->MutantListHead.Blink, \
981 &(Object)->MutantListEntry); \
986 // Satisfies the wait of any nonmutant dispatcher object
988 #define KiSatisfyNonMutantWait(Object) \
990 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
991 EventSynchronizationObject) \
993 /* Synchronization Timers and Events just get un-signaled */ \
994 (Object)->Header.SignalState = 0; \
996 else if ((Object)->Header.Type == SemaphoreObject) \
998 /* These ones can have multiple states, so we only decrease it */ \
999 (Object)->Header.SignalState--; \
1004 // Recalculates the due time
1008 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
1009 IN PLARGE_INTEGER DueTime
,
1010 IN OUT PLARGE_INTEGER NewDueTime
)
1012 /* Don't do anything for absolute waits */
1013 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
1015 /* Otherwise, query the interrupt time and recalculate */
1016 NewDueTime
->QuadPart
= KeQueryInterruptTime();
1017 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
1022 // Determines whether a thread should be added to the wait list
1026 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
1027 IN KPROCESSOR_MODE WaitMode
)
1029 /* Check the required conditions */
1030 if ((WaitMode
!= KernelMode
) &&
1031 (Thread
->EnableStackSwap
) &&
1032 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
1034 /* We are go for swap */
1039 /* Don't swap the thread */
1045 // Adds a thread to the wait list
1047 #define KiAddThreadToWaitList(Thread, Swappable) \
1049 /* Make sure it's swappable */ \
1052 /* Insert it into the PRCB's List */ \
1053 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
1054 &Thread->WaitListEntry); \
1059 // Checks if a wait in progress should be interrupted by APCs or an alertable
1064 KiCheckAlertability(IN PKTHREAD Thread
,
1065 IN BOOLEAN Alertable
,
1066 IN KPROCESSOR_MODE WaitMode
)
1068 /* Check if the wait is alertable */
1071 /* It is, first check if the thread is alerted in this mode */
1072 if (Thread
->Alerted
[WaitMode
])
1074 /* It is, so bail out of the wait */
1075 Thread
->Alerted
[WaitMode
] = FALSE
;
1076 return STATUS_ALERTED
;
1078 else if ((WaitMode
!= KernelMode
) &&
1079 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
1081 /* It's isn't, but this is a user wait with queued user APCs */
1082 Thread
->ApcState
.UserApcPending
= TRUE
;
1083 return STATUS_USER_APC
;
1085 else if (Thread
->Alerted
[KernelMode
])
1087 /* It isn't that either, but we're alered in kernel mode */
1088 Thread
->Alerted
[KernelMode
] = FALSE
;
1089 return STATUS_ALERTED
;
1092 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
1094 /* Not alertable, but this is a user wait with pending user APCs */
1095 return STATUS_USER_APC
;
1098 /* Otherwise, we're fine */
1099 return STATUS_WAIT_0
;
1103 // Called by Wait and Queue code to insert a timer for dispatching.
1104 // Also called by KeSetTimerEx to insert a timer from the caller.
1108 KxInsertTimer(IN PKTIMER Timer
,
1111 PKSPIN_LOCK_QUEUE LockQueue
;
1113 /* Acquire the lock and release the dispatcher lock */
1114 LockQueue
= KiAcquireTimerLock(Hand
);
1115 KiReleaseDispatcherLockFromDpcLevel();
1117 /* Try to insert the timer */
1118 if (KiInsertTimerTable(Timer
, Hand
))
1121 KiCompleteTimer(Timer
, LockQueue
);
1125 /* Do nothing, just release the lock */
1126 KiReleaseTimerLock(LockQueue
);
1131 // Called from Unlink and Queue Insert Code.
1132 // Also called by timer code when canceling an inserted timer.
1133 // Removes a timer from it's tree.
1137 KxRemoveTreeTimer(IN PKTIMER Timer
)
1139 ULONG Hand
= Timer
->Header
.Hand
;
1140 PKSPIN_LOCK_QUEUE LockQueue
;
1141 PKTIMER_TABLE_ENTRY TimerEntry
;
1143 /* Acquire timer lock */
1144 LockQueue
= KiAcquireTimerLock(Hand
);
1146 /* Set the timer as non-inserted */
1147 Timer
->Header
.Inserted
= FALSE
;
1149 /* Remove it from the timer list */
1150 if (RemoveEntryList(&Timer
->TimerListEntry
))
1152 /* Get the entry and check if it's empty */
1153 TimerEntry
= &KiTimerTableListHead
[Hand
];
1154 if (IsListEmpty(&TimerEntry
->Entry
))
1156 /* Clear the time then */
1157 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1161 /* Release the timer lock */
1162 KiReleaseTimerLock(LockQueue
);
1167 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1168 IN LARGE_INTEGER Interval
,
1172 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1174 /* Check the timer's interval to see if it's absolute */
1175 Timer
->Header
.Absolute
= FALSE
;
1176 if (Interval
.HighPart
>= 0)
1178 /* Get the system time and calculate the relative time */
1179 KeQuerySystemTime(&SystemTime
);
1180 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1181 Timer
->Header
.Absolute
= TRUE
;
1183 /* Check if we've already expired */
1184 if (TimeDifference
.HighPart
>= 0)
1186 /* Reset everything */
1187 Timer
->DueTime
.QuadPart
= 0;
1189 Timer
->Header
.Hand
= 0;
1194 /* Update the interval */
1195 Interval
= TimeDifference
;
1199 /* Calculate the due time */
1200 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1201 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1202 Timer
->DueTime
.QuadPart
= DueTime
;
1204 /* Calculate the timer handle */
1205 *Hand
= KiComputeTimerTableIndex(DueTime
);
1206 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1209 #define KxDelayThreadWait() \
1211 /* Setup the Wait Block */ \
1212 Thread->WaitBlockList = TimerBlock; \
1214 /* Setup the timer */ \
1215 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1217 /* Save the due time for the caller */ \
1218 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1220 /* Link the timer to this Wait Block */ \
1221 TimerBlock->NextWaitBlock = TimerBlock; \
1222 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1223 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1225 /* Clear wait status */ \
1226 Thread->WaitStatus = STATUS_SUCCESS; \
1228 /* Setup wait fields */ \
1229 Thread->Alertable = Alertable; \
1230 Thread->WaitReason = DelayExecution; \
1231 Thread->WaitMode = WaitMode; \
1233 /* Check if we can swap the thread's stack */ \
1234 Thread->WaitListEntry.Flink = NULL; \
1235 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1237 /* Set the wait time */ \
1238 Thread->WaitTime = KeTickCount.LowPart;
1240 #define KxMultiThreadWait() \
1241 /* Link wait block array to the thread */ \
1242 Thread->WaitBlockList = WaitBlockArray; \
1244 /* Reset the index */ \
1247 /* Loop wait blocks */ \
1250 /* Fill out the wait block */ \
1251 WaitBlock = &WaitBlockArray[Index]; \
1252 WaitBlock->Object = Object[Index]; \
1253 WaitBlock->WaitKey = (USHORT)Index; \
1254 WaitBlock->WaitType = WaitType; \
1255 WaitBlock->Thread = Thread; \
1257 /* Link to next block */ \
1258 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1260 } while (Index < Count); \
1262 /* Link the last block */ \
1263 WaitBlock->NextWaitBlock = WaitBlockArray; \
1265 /* Set default wait status */ \
1266 Thread->WaitStatus = STATUS_WAIT_0; \
1268 /* Check if we have a timer */ \
1271 /* Link to the block */ \
1272 TimerBlock->NextWaitBlock = WaitBlockArray; \
1274 /* Setup the timer */ \
1275 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1277 /* Save the due time for the caller */ \
1278 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1280 /* Initialize the list */ \
1281 InitializeListHead(&Timer->Header.WaitListHead); \
1284 /* Set wait settings */ \
1285 Thread->Alertable = Alertable; \
1286 Thread->WaitMode = WaitMode; \
1287 Thread->WaitReason = WaitReason; \
1289 /* Check if we can swap the thread's stack */ \
1290 Thread->WaitListEntry.Flink = NULL; \
1291 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1293 /* Set the wait time */ \
1294 Thread->WaitTime = KeTickCount.LowPart;
1296 #define KxSingleThreadWait() \
1297 /* Setup the Wait Block */ \
1298 Thread->WaitBlockList = WaitBlock; \
1299 WaitBlock->WaitKey = STATUS_SUCCESS; \
1300 WaitBlock->Object = Object; \
1301 WaitBlock->WaitType = WaitAny; \
1303 /* Clear wait status */ \
1304 Thread->WaitStatus = STATUS_SUCCESS; \
1306 /* Check if we have a timer */ \
1309 /* Setup the timer */ \
1310 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1312 /* Save the due time for the caller */ \
1313 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1315 /* Pointer to timer block */ \
1316 WaitBlock->NextWaitBlock = TimerBlock; \
1317 TimerBlock->NextWaitBlock = WaitBlock; \
1319 /* Link the timer to this Wait Block */ \
1320 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1321 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1325 /* No timer block, just ourselves */ \
1326 WaitBlock->NextWaitBlock = WaitBlock; \
1329 /* Set wait settings */ \
1330 Thread->Alertable = Alertable; \
1331 Thread->WaitMode = WaitMode; \
1332 Thread->WaitReason = WaitReason; \
1334 /* Check if we can swap the thread's stack */ \
1335 Thread->WaitListEntry.Flink = NULL; \
1336 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1338 /* Set the wait time */ \
1339 Thread->WaitTime = KeTickCount.LowPart;
1341 #define KxQueueThreadWait() \
1342 /* Setup the Wait Block */ \
1343 Thread->WaitBlockList = WaitBlock; \
1344 WaitBlock->WaitKey = STATUS_SUCCESS; \
1345 WaitBlock->Object = Queue; \
1346 WaitBlock->WaitType = WaitAny; \
1347 WaitBlock->Thread = Thread; \
1349 /* Clear wait status */ \
1350 Thread->WaitStatus = STATUS_SUCCESS; \
1352 /* Check if we have a timer */ \
1355 /* Setup the timer */ \
1356 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1358 /* Save the due time for the caller */ \
1359 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1361 /* Pointer to timer block */ \
1362 WaitBlock->NextWaitBlock = TimerBlock; \
1363 TimerBlock->NextWaitBlock = WaitBlock; \
1365 /* Link the timer to this Wait Block */ \
1366 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1367 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1371 /* No timer block, just ourselves */ \
1372 WaitBlock->NextWaitBlock = WaitBlock; \
1375 /* Set wait settings */ \
1376 Thread->Alertable = FALSE; \
1377 Thread->WaitMode = WaitMode; \
1378 Thread->WaitReason = WrQueue; \
1380 /* Check if we can swap the thread's stack */ \
1381 Thread->WaitListEntry.Flink = NULL; \
1382 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1384 /* Set the wait time */ \
1385 Thread->WaitTime = KeTickCount.LowPart;
1392 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1393 IN KPRIORITY Increment
)
1395 PLIST_ENTRY WaitEntry
, WaitList
;
1396 PKWAIT_BLOCK WaitBlock
;
1397 PKTHREAD WaitThread
;
1400 /* Loop the Wait Entries */
1401 WaitList
= &Object
->WaitListHead
;
1402 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1403 WaitEntry
= WaitList
->Flink
;
1406 /* Get the current wait block */
1407 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1409 /* Get the waiting thread */
1410 WaitThread
= WaitBlock
->Thread
;
1412 /* Check the current Wait Mode */
1413 if (WaitBlock
->WaitType
== WaitAny
)
1415 /* Use the actual wait key */
1416 WaitKey
= WaitBlock
->WaitKey
;
1420 /* Otherwise, use STATUS_KERNEL_APC */
1421 WaitKey
= STATUS_KERNEL_APC
;
1424 /* Unwait the thread */
1425 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1428 WaitEntry
= WaitList
->Flink
;
1429 } while (WaitEntry
!= WaitList
);
1433 // Unwaits a Thread waiting on an event
1437 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1438 IN KPRIORITY Increment
)
1440 PLIST_ENTRY WaitEntry
, WaitList
;
1441 PKWAIT_BLOCK WaitBlock
;
1442 PKTHREAD WaitThread
;
1444 /* Loop the Wait Entries */
1445 WaitList
= &Event
->Header
.WaitListHead
;
1446 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1447 WaitEntry
= WaitList
->Flink
;
1450 /* Get the current wait block */
1451 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1453 /* Get the waiting thread */
1454 WaitThread
= WaitBlock
->Thread
;
1456 /* Check the current Wait Mode */
1457 if (WaitBlock
->WaitType
== WaitAny
)
1460 Event
->Header
.SignalState
= 0;
1462 /* Un-signal the event and unwait the thread */
1463 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1467 /* Unwait the thread with STATUS_KERNEL_APC */
1468 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1471 WaitEntry
= WaitList
->Flink
;
1472 } while (WaitEntry
!= WaitList
);
1476 // This routine queues a thread that is ready on the PRCB's ready lists.
1477 // If this thread cannot currently run on this CPU, then the thread is
1478 // added to the deferred ready list instead.
1480 // This routine must be entered with the PRCB lock held and it will exit
1481 // with the PRCB lock released!
1485 KxQueueReadyThread(IN PKTHREAD Thread
,
1492 ASSERT(Prcb
== KeGetCurrentPrcb());
1493 ASSERT(Thread
->State
== Running
);
1494 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1496 /* Check if this thread is allowed to run in this CPU */
1498 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1503 /* Set thread ready for execution */
1504 Thread
->State
= Ready
;
1506 /* Save current priority and if someone had pre-empted it */
1507 Priority
= Thread
->Priority
;
1508 Preempted
= Thread
->Preempted
;
1510 /* We're not pre-empting now, and set the wait time */
1511 Thread
->Preempted
= FALSE
;
1512 Thread
->WaitTime
= KeTickCount
.LowPart
;
1515 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1517 /* Insert this thread in the appropriate order */
1518 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1519 &Thread
->WaitListEntry
) :
1520 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1521 &Thread
->WaitListEntry
);
1523 /* Update the ready summary */
1524 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1527 ASSERT(Priority
== Thread
->Priority
);
1529 /* Release the PRCB lock */
1530 KiReleasePrcbLock(Prcb
);
1534 /* Otherwise, prepare this thread to be deferred */
1535 Thread
->State
= DeferredReady
;
1536 Thread
->DeferredProcessor
= Prcb
->Number
;
1538 /* Release the lock and defer scheduling */
1539 KiReleasePrcbLock(Prcb
);
1540 KiDeferredReadyThread(Thread
);
1545 // This routine scans for an appropriate ready thread to select at the
1546 // given priority and for the given CPU.
1550 KiSelectReadyThread(IN KPRIORITY Priority
,
1553 ULONG PrioritySet
, HighPriority
;
1554 PLIST_ENTRY ListEntry
;
1555 PKTHREAD Thread
= NULL
;
1557 /* Save the current mask and get the priority set for the CPU */
1558 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1559 if (!PrioritySet
) goto Quickie
;
1561 /* Get the highest priority possible */
1562 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1563 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1564 HighPriority
+= Priority
;
1566 /* Make sure the list isn't empty at the highest priority */
1567 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1569 /* Get the first thread on the list */
1570 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1571 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1573 /* Make sure this thread is here for a reason */
1574 ASSERT(HighPriority
== Thread
->Priority
);
1575 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1576 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1578 /* Remove it from the list */
1579 if (RemoveEntryList(&Thread
->WaitListEntry
))
1581 /* The list is empty now, reset the ready summary */
1582 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1585 /* Sanity check and return the thread */
1587 ASSERT((Thread
== NULL
) ||
1588 (Thread
->BasePriority
== 0) ||
1589 (Thread
->Priority
!= 0));
1594 // This routine computes the new priority for a thread. It is only valid for
1595 // threads with priorities in the dynamic priority range.
1599 KiComputeNewPriority(IN PKTHREAD Thread
,
1600 IN SCHAR Adjustment
)
1604 /* Priority sanity checks */
1605 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1606 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1607 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1608 TRUE
: (Thread
->PriorityDecrement
== 0));
1610 /* Get the current priority */
1611 Priority
= Thread
->Priority
;
1612 if (Priority
< LOW_REALTIME_PRIORITY
)
1614 /* Decrease priority by the priority decrement */
1615 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1617 /* Don't go out of bounds */
1618 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1620 /* Reset the priority decrement */
1621 Thread
->PriorityDecrement
= 0;
1625 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1627 /* Return the new priority */