2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
17 // Sanitizes a selector
21 Ke386SanitizeSeg(IN ULONG Cs
,
22 IN KPROCESSOR_MODE Mode
)
25 // Check if we're in kernel-mode, and force CPL 0 if so.
26 // Otherwise, force CPL 3.
28 return ((Mode
== KernelMode
) ?
29 (Cs
& (0xFFFF & ~RPL_MASK
)) :
30 (RPL_MASK
| (Cs
& 0xFFFF)));
38 Ke386SanitizeFlags(IN ULONG Eflags
,
39 IN KPROCESSOR_MODE Mode
)
42 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
43 // Otherwise, also force interrupt mask on.
45 return ((Mode
== KernelMode
) ?
46 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
47 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
51 // Gets a DR register from a CONTEXT structure
55 KiDrFromContext(IN ULONG Dr
,
58 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
62 // Gets a DR register from a KTRAP_FRAME structure
66 KiDrFromTrapFrame(IN ULONG Dr
,
67 IN PKTRAP_FRAME TrapFrame
)
69 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
77 Ke386SanitizeDr(IN PVOID DrAddress
,
78 IN KPROCESSOR_MODE Mode
)
81 // Check if we're in kernel-mode, and return the address directly if so.
82 // Otherwise, make sure it's not inside the kernel-mode address space.
83 // If it is, then clear the address.
85 return ((Mode
== KernelMode
) ? DrAddress
:
86 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
90 // Enters a Guarded Region
92 #define KeEnterGuardedRegion() \
94 PKTHREAD _Thread = KeGetCurrentThread(); \
97 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
98 ASSERT(_Thread == KeGetCurrentThread()); \
99 ASSERT((_Thread->SpecialApcDisable <= 0) && \
100 (_Thread->SpecialApcDisable != -32768)); \
102 /* Disable Special APCs */ \
103 _Thread->SpecialApcDisable--; \
107 // Leaves a Guarded Region
109 #define KeLeaveGuardedRegion() \
111 PKTHREAD _Thread = KeGetCurrentThread(); \
113 /* Sanity checks */ \
114 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
115 ASSERT(_Thread == KeGetCurrentThread()); \
116 ASSERT(_Thread->SpecialApcDisable < 0); \
118 /* Leave region and check if APCs are OK now */ \
119 if (!(++_Thread->SpecialApcDisable)) \
121 /* Check for Kernel APCs on the list */ \
122 if (!IsListEmpty(&_Thread->ApcState. \
123 ApcListHead[KernelMode])) \
125 /* Check for APC Delivery */ \
126 KiCheckForKernelApcDelivery(); \
132 // TODO: Guarded Mutex Routines
136 // Enters a Critical Region
138 #define KeEnterCriticalRegion() \
140 PKTHREAD _Thread = KeGetCurrentThread(); \
142 /* Sanity checks */ \
143 ASSERT(_Thread == KeGetCurrentThread()); \
144 ASSERT((_Thread->KernelApcDisable <= 0) && \
145 (_Thread->KernelApcDisable != -32768)); \
147 /* Disable Kernel APCs */ \
148 _Thread->KernelApcDisable--; \
152 // Leaves a Critical Region
154 #define KeLeaveCriticalRegion() \
156 PKTHREAD _Thread = KeGetCurrentThread(); \
158 /* Sanity checks */ \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->KernelApcDisable < 0); \
162 /* Enable Kernel APCs */ \
163 _Thread->KernelApcDisable++; \
165 /* Check if Kernel APCs are now enabled */ \
166 if (!(_Thread->KernelApcDisable)) \
168 /* Check if we need to request an APC Delivery */ \
169 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
170 !(_Thread->SpecialApcDisable)) \
172 /* Check for the right environment */ \
173 KiCheckForKernelApcDelivery(); \
180 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
184 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
186 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
187 UNREFERENCED_PARAMETER(SpinLock
);
191 // Spinlock Release at IRQL >= DISPATCH_LEVEL
195 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
197 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
198 UNREFERENCED_PARAMETER(SpinLock
);
202 // This routine protects against multiple CPU acquires, it's meaningless on UP.
206 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
208 UNREFERENCED_PARAMETER(Object
);
212 // This routine protects against multiple CPU acquires, it's meaningless on UP.
216 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
218 UNREFERENCED_PARAMETER(Object
);
223 KiAcquireDispatcherLock(VOID
)
225 /* Raise to DPC level */
226 return KeRaiseIrqlToDpcLevel();
231 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
233 /* Just exit the dispatcher */
234 KiExitDispatcher(OldIrql
);
239 KiAcquireDispatcherLockAtDpcLevel(VOID
)
241 /* This is a no-op at DPC Level for UP systems */
247 KiReleaseDispatcherLockFromDpcLevel(VOID
)
249 /* This is a no-op at DPC Level for UP systems */
254 // This routine makes the thread deferred ready on the boot CPU.
258 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
260 /* Set the thread to deferred state and boot CPU */
261 Thread
->State
= DeferredReady
;
262 Thread
->DeferredProcessor
= 0;
264 /* Make the thread ready immediately */
265 KiDeferredReadyThread(Thread
);
270 KiRescheduleThread(IN BOOLEAN NewThread
,
273 /* This is meaningless on UP systems */
274 UNREFERENCED_PARAMETER(NewThread
);
275 UNREFERENCED_PARAMETER(Cpu
);
279 // This routine protects against multiple CPU acquires, it's meaningless on UP.
283 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
285 UNREFERENCED_PARAMETER(Thread
);
289 // This routine protects against multiple CPU acquires, it's meaningless on UP.
293 KiAcquirePrcbLock(IN PKPRCB Prcb
)
295 UNREFERENCED_PARAMETER(Prcb
);
299 // This routine protects against multiple CPU acquires, it's meaningless on UP.
303 KiReleasePrcbLock(IN PKPRCB Prcb
)
305 UNREFERENCED_PARAMETER(Prcb
);
309 // This routine protects against multiple CPU acquires, it's meaningless on UP.
313 KiAcquireThreadLock(IN PKTHREAD Thread
)
315 UNREFERENCED_PARAMETER(Thread
);
319 // This routine protects against multiple CPU acquires, it's meaningless on UP.
323 KiReleaseThreadLock(IN PKTHREAD Thread
)
325 UNREFERENCED_PARAMETER(Thread
);
329 // This routine protects against multiple CPU acquires, it's meaningless on UP.
333 KiTryThreadLock(IN PKTHREAD Thread
)
335 UNREFERENCED_PARAMETER(Thread
);
341 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
343 /* There are no deferred ready lists on UP systems */
344 UNREFERENCED_PARAMETER(Prcb
);
349 KiRundownThread(IN PKTHREAD Thread
)
351 /* Check if this is the NPX Thread */
352 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
355 KeGetCurrentPrcb()->NpxThread
= NULL
;
362 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
365 /* We deliver instantly on UP */
366 UNREFERENCED_PARAMETER(NeedApc
);
367 UNREFERENCED_PARAMETER(Processor
);
372 KiAcquireTimerLock(IN ULONG Hand
)
374 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
376 /* Nothing to do on UP */
377 UNREFERENCED_PARAMETER(Hand
);
383 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
385 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
387 /* Nothing to do on UP */
388 UNREFERENCED_PARAMETER(LockQueue
);
394 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
398 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
402 /* Try to acquire it */
403 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
405 /* Value changed... wait until it's locked */
406 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
409 /* On debug builds, we use a much slower but useful routine */
410 Kii386SpinOnSpinLock(SpinLock
, 5);
412 /* Otherwise, just yield and keep looping */
420 /* On debug builds, we OR in the KTHREAD */
421 *SpinLock
= KeGetCurrentThread() | 1;
423 /* All is well, break out */
430 // Spinlock Release at IRQL >= DISPATCH_LEVEL
434 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
437 /* Make sure that the threads match */
438 if ((KeGetCurrentThread() | 1) != *SpinLock
)
440 /* They don't, bugcheck */
441 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, SpinLock
, 0, 0, 0);
445 InterlockedAnd(SpinLock
, 0);
450 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
452 LONG OldValue
, NewValue
;
454 /* Make sure we're at a safe level to touch the lock */
455 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
457 /* Start acquire loop */
460 /* Loop until the other CPU releases it */
461 while ((UCHAR
)Object
->Lock
& KOBJECT_LOCK_BIT
)
463 /* Let the CPU know that this is a loop */
467 /* Try acquiring the lock now */
468 NewValue
= InterlockedCompareExchange(&Object
->Lock
,
469 OldValue
| KOBJECT_LOCK_BIT
,
471 } while (NewValue
!= OldValue
);
476 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
478 /* Make sure we're at a safe level to touch the lock */
479 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
482 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
487 KiAcquireDispatcherLock(VOID
)
489 /* Raise to synchronization level and acquire the dispatcher lock */
490 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
495 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
497 /* First release the lock */
498 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
499 LockQueue
[LockQueueDispatcherLock
]);
501 /* Then exit the dispatcher */
502 KiExitDispatcher(OldIrql
);
506 // This routine inserts a thread into the deferred ready list of the given CPU
510 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
512 PKPRCB Prcb
= KeGetCurrentPrcb();
514 /* Set the thread to deferred state and CPU */
515 Thread
->State
= DeferredReady
;
516 Thread
->DeferredProcessor
= Prcb
->Number
;
518 /* Add it on the list */
519 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
524 KiRescheduleThread(IN BOOLEAN NewThread
,
527 /* Check if a new thread needs to be scheduled on a different CPU */
528 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
530 /* Send an IPI to request delivery */
531 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
536 // This routine sets the current thread in a swap busy state, which ensure that
537 // nobody else tries to swap it concurrently.
541 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
543 /* Make sure nobody already set it */
544 ASSERT(Thread
->SwapBusy
== FALSE
);
546 /* Set it ourselves */
547 Thread
->SwapBusy
= TRUE
;
551 // This routine acquires the PRCB lock so that only one caller can touch
552 // volatile PRCB data.
554 // Since this is a simple optimized spin-lock, it must be be only acquired
555 // at dispatcher level or higher!
559 KiAcquirePrcbLock(IN PKPRCB Prcb
)
561 /* Make sure we're at a safe level to touch the PRCB lock */
562 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
564 /* Start acquire loop */
567 /* Acquire the lock and break out if we acquired it first */
568 if (!InterlockedExchange(&Prcb
->PrcbLock
, 1)) break;
570 /* Loop until the other CPU releases it */
573 /* Let the CPU know that this is a loop */
575 } while (Prcb
->PrcbLock
);
580 // This routine releases the PRCB lock so that other callers can touch
581 // volatile PRCB data.
583 // Since this is a simple optimized spin-lock, it must be be only acquired
584 // at dispatcher level or higher!
588 KiReleasePrcbLock(IN PKPRCB Prcb
)
590 /* Make sure it's acquired! */
591 ASSERT(Prcb
->PrcbLock
!= 0);
594 InterlockedAnd(&Prcb
->PrcbLock
, 0);
598 // This routine acquires the thread lock so that only one caller can touch
599 // volatile thread data.
601 // Since this is a simple optimized spin-lock, it must be be only acquired
602 // at dispatcher level or higher!
606 KiAcquireThreadLock(IN PKTHREAD Thread
)
608 /* Make sure we're at a safe level to touch the thread lock */
609 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
611 /* Start acquire loop */
614 /* Acquire the lock and break out if we acquired it first */
615 if (!InterlockedExchange(&Thread
->ThreadLock
, 1)) break;
617 /* Loop until the other CPU releases it */
620 /* Let the CPU know that this is a loop */
622 } while (Thread
->ThreadLock
);
627 // This routine releases the thread lock so that other callers can touch
628 // volatile thread data.
630 // Since this is a simple optimized spin-lock, it must be be only acquired
631 // at dispatcher level or higher!
635 KiReleaseThreadLock(IN PKTHREAD Thread
)
638 InterlockedAnd(&Thread
->ThreadLock
, 0);
643 KiTryThreadLock(IN PKTHREAD Thread
)
647 /* If the lock isn't acquired, return false */
648 if (!Thread
->ThreadLock
) return FALSE
;
650 /* Otherwise, try to acquire it and check the result */
652 Value
= InterlockedExchange(&Thread
->ThreadLock
, &Value
);
654 /* Return the lock state */
655 return (Value
== TRUE
);
660 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
662 /* Scan the deferred ready lists if required */
663 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
668 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
671 /* Check if we need to request APC delivery */
674 /* Check if it's on another CPU */
675 if (KeGetPcr()->Number
!= Cpu
)
677 /* Send an IPI to request delivery */
678 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
682 /* Request a software interrupt */
683 HalRequestSoftwareInterrupt(APC_LEVEL
);
692 KiAcquireApcLock(IN PKTHREAD Thread
,
693 IN PKLOCK_QUEUE_HANDLE Handle
)
695 /* Acquire the lock and raise to synchronization level */
696 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
701 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
702 IN PKLOCK_QUEUE_HANDLE Handle
)
704 /* Acquire the lock */
705 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
710 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
711 IN PKLOCK_QUEUE_HANDLE Handle
)
713 /* Acquire the lock */
714 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
719 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
721 /* Release the lock */
722 KeReleaseInStackQueuedSpinLock(Handle
);
727 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
729 /* Release the lock */
730 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
735 KiAcquireProcessLock(IN PKPROCESS Process
,
736 IN PKLOCK_QUEUE_HANDLE Handle
)
738 /* Acquire the lock and raise to synchronization level */
739 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
744 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
746 /* Release the lock */
747 KeReleaseInStackQueuedSpinLock(Handle
);
752 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
754 /* Release the lock */
755 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
760 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
761 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
763 /* Check if we were called from a threaded DPC */
764 if (KeGetCurrentPrcb()->DpcThreadActive
)
766 /* Lock the Queue, we're not at DPC level */
767 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
771 /* We must be at DPC level, acquire the lock safely */
772 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
773 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
780 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
782 /* Check if we were called from a threaded DPC */
783 if (KeGetCurrentPrcb()->DpcThreadActive
)
785 /* Unlock the Queue, we're not at DPC level */
786 KeReleaseInStackQueuedSpinLock(DeviceLock
);
790 /* We must be at DPC level, release the lock safely */
791 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
792 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
797 // Satisfies the wait of any dispatcher object
799 #define KiSatisfyObjectWait(Object, Thread) \
801 /* Special case for Mutants */ \
802 if ((Object)->Header.Type == MutantObject) \
804 /* Decrease the Signal State */ \
805 (Object)->Header.SignalState--; \
807 /* Check if it's now non-signaled */ \
808 if (!(Object)->Header.SignalState) \
810 /* Set the Owner Thread */ \
811 (Object)->OwnerThread = Thread; \
813 /* Disable APCs if needed */ \
814 Thread->KernelApcDisable = Thread->KernelApcDisable - \
815 (Object)->ApcDisable; \
817 /* Check if it's abandoned */ \
818 if ((Object)->Abandoned) \
821 (Object)->Abandoned = FALSE; \
823 /* Return Status */ \
824 Thread->WaitStatus = STATUS_ABANDONED; \
827 /* Insert it into the Mutant List */ \
828 InsertHeadList(Thread->MutantListHead.Blink, \
829 &(Object)->MutantListEntry); \
832 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
833 EventSynchronizationObject) \
835 /* Synchronization Timers and Events just get un-signaled */ \
836 (Object)->Header.SignalState = 0; \
838 else if ((Object)->Header.Type == SemaphoreObject) \
840 /* These ones can have multiple states, so we only decrease it */ \
841 (Object)->Header.SignalState--; \
846 // Satisfies the wait of a mutant dispatcher object
848 #define KiSatisfyMutantWait(Object, Thread) \
850 /* Decrease the Signal State */ \
851 (Object)->Header.SignalState--; \
853 /* Check if it's now non-signaled */ \
854 if (!(Object)->Header.SignalState) \
856 /* Set the Owner Thread */ \
857 (Object)->OwnerThread = Thread; \
859 /* Disable APCs if needed */ \
860 Thread->KernelApcDisable = Thread->KernelApcDisable - \
861 (Object)->ApcDisable; \
863 /* Check if it's abandoned */ \
864 if ((Object)->Abandoned) \
867 (Object)->Abandoned = FALSE; \
869 /* Return Status */ \
870 Thread->WaitStatus = STATUS_ABANDONED; \
873 /* Insert it into the Mutant List */ \
874 InsertHeadList(Thread->MutantListHead.Blink, \
875 &(Object)->MutantListEntry); \
880 // Satisfies the wait of any nonmutant dispatcher object
882 #define KiSatisfyNonMutantWait(Object) \
884 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
885 EventSynchronizationObject) \
887 /* Synchronization Timers and Events just get un-signaled */ \
888 (Object)->Header.SignalState = 0; \
890 else if ((Object)->Header.Type == SemaphoreObject) \
892 /* These ones can have multiple states, so we only decrease it */ \
893 (Object)->Header.SignalState--; \
898 // Recalculates the due time
902 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
903 IN PLARGE_INTEGER DueTime
,
904 IN OUT PLARGE_INTEGER NewDueTime
)
906 /* Don't do anything for absolute waits */
907 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
909 /* Otherwise, query the interrupt time and recalculate */
910 NewDueTime
->QuadPart
= KeQueryInterruptTime();
911 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
916 // Determines whether a thread should be added to the wait list
920 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
921 IN KPROCESSOR_MODE WaitMode
)
923 /* Check the required conditions */
924 if ((WaitMode
!= KernelMode
) &&
925 (Thread
->EnableStackSwap
) &&
926 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
928 /* We are go for swap */
933 /* Don't swap the thread */
939 // Adds a thread to the wait list
941 #define KiAddThreadToWaitList(Thread, Swappable) \
943 /* Make sure it's swappable */ \
946 /* Insert it into the PRCB's List */ \
947 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
948 &Thread->WaitListEntry); \
953 // Checks if a wait in progress should be interrupted by APCs or an alertable
958 KiCheckAlertability(IN PKTHREAD Thread
,
959 IN BOOLEAN Alertable
,
960 IN KPROCESSOR_MODE WaitMode
)
962 /* Check if the wait is alertable */
965 /* It is, first check if the thread is alerted in this mode */
966 if (Thread
->Alerted
[WaitMode
])
968 /* It is, so bail out of the wait */
969 Thread
->Alerted
[WaitMode
] = FALSE
;
970 return STATUS_ALERTED
;
972 else if ((WaitMode
!= KernelMode
) &&
973 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
975 /* It's isn't, but this is a user wait with queued user APCs */
976 Thread
->ApcState
.UserApcPending
= TRUE
;
977 return STATUS_USER_APC
;
979 else if (Thread
->Alerted
[KernelMode
])
981 /* It isn't that either, but we're alered in kernel mode */
982 Thread
->Alerted
[KernelMode
] = FALSE
;
983 return STATUS_ALERTED
;
986 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
988 /* Not alertable, but this is a user wait with pending user APCs */
989 return STATUS_USER_APC
;
992 /* Otherwise, we're fine */
993 return STATUS_WAIT_0
;
997 // Called by Wait and Queue code to insert a timer for dispatching.
998 // Also called by KeSetTimerEx to insert a timer from the caller.
1002 KxInsertTimer(IN PKTIMER Timer
,
1005 PKSPIN_LOCK_QUEUE LockQueue
;
1007 /* Acquire the lock and release the dispatcher lock */
1008 LockQueue
= KiAcquireTimerLock(Hand
);
1009 KiReleaseDispatcherLockFromDpcLevel();
1011 /* Try to insert the timer */
1012 if (KiInsertTimerTable(Timer
, Hand
))
1015 KiCompleteTimer(Timer
, LockQueue
);
1019 /* Do nothing, just release the lock */
1020 KiReleaseTimerLock(LockQueue
);
1025 // Called from Unlink and Queue Insert Code.
1026 // Also called by timer code when canceling an inserted timer.
1027 // Removes a timer from it's tree.
1031 KxRemoveTreeTimer(IN PKTIMER Timer
)
1033 ULONG Hand
= Timer
->Header
.Hand
;
1034 PKSPIN_LOCK_QUEUE LockQueue
;
1035 PKTIMER_TABLE_ENTRY TimerEntry
;
1037 /* Acquire timer lock */
1038 LockQueue
= KiAcquireTimerLock(Hand
);
1040 /* Set the timer as non-inserted */
1041 Timer
->Header
.Inserted
= FALSE
;
1043 /* Remove it from the timer list */
1044 if (RemoveEntryList(&Timer
->TimerListEntry
))
1046 /* Get the entry and check if it's empty */
1047 TimerEntry
= &KiTimerTableListHead
[Hand
];
1048 if (IsListEmpty(&TimerEntry
->Entry
))
1050 /* Clear the time then */
1051 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1055 /* Release the timer lock */
1056 KiReleaseTimerLock(LockQueue
);
1061 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1062 IN LARGE_INTEGER Interval
,
1066 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1068 /* Check the timer's interval to see if it's absolute */
1069 Timer
->Header
.Absolute
= FALSE
;
1070 if (Interval
.HighPart
>= 0)
1072 /* Get the system time and calculate the relative time */
1073 KeQuerySystemTime(&SystemTime
);
1074 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1075 Timer
->Header
.Absolute
= TRUE
;
1077 /* Check if we've already expired */
1078 if (TimeDifference
.HighPart
>= 0)
1080 /* Reset everything */
1081 Timer
->DueTime
.QuadPart
= 0;
1083 Timer
->Header
.Hand
= 0;
1088 /* Update the interval */
1089 Interval
= TimeDifference
;
1093 /* Calculate the due time */
1094 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1095 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1096 Timer
->DueTime
.QuadPart
= DueTime
;
1098 /* Calculate the timer handle */
1099 *Hand
= KiComputeTimerTableIndex(DueTime
);
1100 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1103 #define KxDelayThreadWait() \
1105 /* Setup the Wait Block */ \
1106 Thread->WaitBlockList = TimerBlock; \
1108 /* Setup the timer */ \
1109 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1111 /* Save the due time for the caller */ \
1112 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1114 /* Link the timer to this Wait Block */ \
1115 TimerBlock->NextWaitBlock = TimerBlock; \
1116 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1117 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1119 /* Clear wait status */ \
1120 Thread->WaitStatus = STATUS_SUCCESS; \
1122 /* Setup wait fields */ \
1123 Thread->Alertable = Alertable; \
1124 Thread->WaitReason = DelayExecution; \
1125 Thread->WaitMode = WaitMode; \
1127 /* Check if we can swap the thread's stack */ \
1128 Thread->WaitListEntry.Flink = NULL; \
1129 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1131 /* Set the wait time */ \
1132 Thread->WaitTime = KeTickCount.LowPart;
1134 #define KxMultiThreadWait() \
1135 /* Link wait block array to the thread */ \
1136 Thread->WaitBlockList = WaitBlockArray; \
1138 /* Reset the index */ \
1141 /* Loop wait blocks */ \
1144 /* Fill out the wait block */ \
1145 WaitBlock = &WaitBlockArray[Index]; \
1146 WaitBlock->Object = Object[Index]; \
1147 WaitBlock->WaitKey = (USHORT)Index; \
1148 WaitBlock->WaitType = WaitType; \
1149 WaitBlock->Thread = Thread; \
1151 /* Link to next block */ \
1152 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1154 } while (Index < Count); \
1156 /* Link the last block */ \
1157 WaitBlock->NextWaitBlock = WaitBlockArray; \
1159 /* Set default wait status */ \
1160 Thread->WaitStatus = STATUS_WAIT_0; \
1162 /* Check if we have a timer */ \
1165 /* Link to the block */ \
1166 TimerBlock->NextWaitBlock = WaitBlockArray; \
1168 /* Setup the timer */ \
1169 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1171 /* Save the due time for the caller */ \
1172 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1174 /* Initialize the list */ \
1175 InitializeListHead(&Timer->Header.WaitListHead); \
1178 /* Set wait settings */ \
1179 Thread->Alertable = Alertable; \
1180 Thread->WaitMode = WaitMode; \
1181 Thread->WaitReason = WaitReason; \
1183 /* Check if we can swap the thread's stack */ \
1184 Thread->WaitListEntry.Flink = NULL; \
1185 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1187 /* Set the wait time */ \
1188 Thread->WaitTime = KeTickCount.LowPart;
1190 #define KxSingleThreadWait() \
1191 /* Setup the Wait Block */ \
1192 Thread->WaitBlockList = WaitBlock; \
1193 WaitBlock->WaitKey = STATUS_SUCCESS; \
1194 WaitBlock->Object = Object; \
1195 WaitBlock->WaitType = WaitAny; \
1197 /* Clear wait status */ \
1198 Thread->WaitStatus = STATUS_SUCCESS; \
1200 /* Check if we have a timer */ \
1203 /* Setup the timer */ \
1204 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1206 /* Save the due time for the caller */ \
1207 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1209 /* Pointer to timer block */ \
1210 WaitBlock->NextWaitBlock = TimerBlock; \
1211 TimerBlock->NextWaitBlock = WaitBlock; \
1213 /* Link the timer to this Wait Block */ \
1214 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1215 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1219 /* No timer block, just ourselves */ \
1220 WaitBlock->NextWaitBlock = WaitBlock; \
1223 /* Set wait settings */ \
1224 Thread->Alertable = Alertable; \
1225 Thread->WaitMode = WaitMode; \
1226 Thread->WaitReason = WaitReason; \
1228 /* Check if we can swap the thread's stack */ \
1229 Thread->WaitListEntry.Flink = NULL; \
1230 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1232 /* Set the wait time */ \
1233 Thread->WaitTime = KeTickCount.LowPart;
1235 #define KxQueueThreadWait() \
1236 /* Setup the Wait Block */ \
1237 Thread->WaitBlockList = WaitBlock; \
1238 WaitBlock->WaitKey = STATUS_SUCCESS; \
1239 WaitBlock->Object = Queue; \
1240 WaitBlock->WaitType = WaitAny; \
1241 WaitBlock->Thread = Thread; \
1243 /* Clear wait status */ \
1244 Thread->WaitStatus = STATUS_SUCCESS; \
1246 /* Check if we have a timer */ \
1249 /* Setup the timer */ \
1250 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1252 /* Save the due time for the caller */ \
1253 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1255 /* Pointer to timer block */ \
1256 WaitBlock->NextWaitBlock = TimerBlock; \
1257 TimerBlock->NextWaitBlock = WaitBlock; \
1259 /* Link the timer to this Wait Block */ \
1260 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1261 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1265 /* No timer block, just ourselves */ \
1266 WaitBlock->NextWaitBlock = WaitBlock; \
1269 /* Set wait settings */ \
1270 Thread->Alertable = FALSE; \
1271 Thread->WaitMode = WaitMode; \
1272 Thread->WaitReason = WrQueue; \
1274 /* Check if we can swap the thread's stack */ \
1275 Thread->WaitListEntry.Flink = NULL; \
1276 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1278 /* Set the wait time */ \
1279 Thread->WaitTime = KeTickCount.LowPart;
1286 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1287 IN KPRIORITY Increment
)
1289 PLIST_ENTRY WaitEntry
, WaitList
;
1290 PKWAIT_BLOCK WaitBlock
;
1291 PKTHREAD WaitThread
;
1294 /* Loop the Wait Entries */
1295 WaitList
= &Object
->WaitListHead
;
1296 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1297 WaitEntry
= WaitList
->Flink
;
1300 /* Get the current wait block */
1301 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1303 /* Get the waiting thread */
1304 WaitThread
= WaitBlock
->Thread
;
1306 /* Check the current Wait Mode */
1307 if (WaitBlock
->WaitType
== WaitAny
)
1309 /* Use the actual wait key */
1310 WaitKey
= WaitBlock
->WaitKey
;
1314 /* Otherwise, use STATUS_KERNEL_APC */
1315 WaitKey
= STATUS_KERNEL_APC
;
1318 /* Unwait the thread */
1319 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1322 WaitEntry
= WaitList
->Flink
;
1323 } while (WaitEntry
!= WaitList
);
1327 // Unwaits a Thread waiting on an event
1331 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1332 IN KPRIORITY Increment
)
1334 PLIST_ENTRY WaitEntry
, WaitList
;
1335 PKWAIT_BLOCK WaitBlock
;
1336 PKTHREAD WaitThread
;
1338 /* Loop the Wait Entries */
1339 WaitList
= &Event
->Header
.WaitListHead
;
1340 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1341 WaitEntry
= WaitList
->Flink
;
1344 /* Get the current wait block */
1345 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1347 /* Get the waiting thread */
1348 WaitThread
= WaitBlock
->Thread
;
1350 /* Check the current Wait Mode */
1351 if (WaitBlock
->WaitType
== WaitAny
)
1354 Event
->Header
.SignalState
= 0;
1356 /* Un-signal the event and unwait the thread */
1357 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1361 /* Unwait the thread with STATUS_KERNEL_APC */
1362 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1365 WaitEntry
= WaitList
->Flink
;
1366 } while (WaitEntry
!= WaitList
);
1370 // This routine queues a thread that is ready on the PRCB's ready lists.
1371 // If this thread cannot currently run on this CPU, then the thread is
1372 // added to the deferred ready list instead.
1374 // This routine must be entered with the PRCB lock held and it will exit
1375 // with the PRCB lock released!
1379 KxQueueReadyThread(IN PKTHREAD Thread
,
1386 ASSERT(Prcb
== KeGetCurrentPrcb());
1387 ASSERT(Thread
->State
== Running
);
1388 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1390 /* Check if this thread is allowed to run in this CPU */
1392 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1397 /* Set thread ready for execution */
1398 Thread
->State
= Ready
;
1400 /* Save current priority and if someone had pre-empted it */
1401 Priority
= Thread
->Priority
;
1402 Preempted
= Thread
->Preempted
;
1404 /* We're not pre-empting now, and set the wait time */
1405 Thread
->Preempted
= FALSE
;
1406 Thread
->WaitTime
= KeTickCount
.LowPart
;
1409 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1411 /* Insert this thread in the appropriate order */
1412 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1413 &Thread
->WaitListEntry
) :
1414 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1415 &Thread
->WaitListEntry
);
1417 /* Update the ready summary */
1418 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1421 ASSERT(Priority
== Thread
->Priority
);
1423 /* Release the PRCB lock */
1424 KiReleasePrcbLock(Prcb
);
1428 /* Otherwise, prepare this thread to be deferred */
1429 Thread
->State
= DeferredReady
;
1430 Thread
->DeferredProcessor
= Prcb
->Number
;
1432 /* Release the lock and defer scheduling */
1433 KiReleasePrcbLock(Prcb
);
1434 KiDeferredReadyThread(Thread
);
1439 // This routine scans for an appropriate ready thread to select at the
1440 // given priority and for the given CPU.
1444 KiSelectReadyThread(IN KPRIORITY Priority
,
1447 ULONG PrioritySet
, HighPriority
;
1448 PLIST_ENTRY ListEntry
;
1449 PKTHREAD Thread
= NULL
;
1451 /* Save the current mask and get the priority set for the CPU */
1452 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1453 if (!PrioritySet
) goto Quickie
;
1455 /* Get the highest priority possible */
1456 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1457 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1458 HighPriority
+= Priority
;
1460 /* Make sure the list isn't empty at the highest priority */
1461 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1463 /* Get the first thread on the list */
1464 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1465 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1467 /* Make sure this thread is here for a reason */
1468 ASSERT(HighPriority
== Thread
->Priority
);
1469 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1470 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1472 /* Remove it from the list */
1473 if (RemoveEntryList(&Thread
->WaitListEntry
))
1475 /* The list is empty now, reset the ready summary */
1476 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1479 /* Sanity check and return the thread */
1481 ASSERT((Thread
== NULL
) ||
1482 (Thread
->BasePriority
== 0) ||
1483 (Thread
->Priority
!= 0));
1488 // This routine computes the new priority for a thread. It is only valid for
1489 // threads with priorities in the dynamic priority range.
1493 KiComputeNewPriority(IN PKTHREAD Thread
,
1494 IN SCHAR Adjustment
)
1498 /* Priority sanity checks */
1499 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1500 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1501 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1502 TRUE
: (Thread
->PriorityDecrement
== 0));
1504 /* Get the current priority */
1505 Priority
= Thread
->Priority
;
1506 if (Priority
< LOW_REALTIME_PRIORITY
)
1508 /* Decrease priority by the priority decrement */
1509 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1511 /* Don't go out of bounds */
1512 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1514 /* Reset the priority decrement */
1515 Thread
->PriorityDecrement
= 0;
1519 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1521 /* Return the new priority */
1527 KeGetCurrentThread(VOID
)
1529 /* Return the current thread */
1530 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
1535 KeGetPreviousMode(VOID
)
1537 /* Return the current mode */
1538 return KeGetCurrentThread()->PreviousMode
;