2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
18 // Sanitizes a selector
22 Ke386SanitizeSeg(IN ULONG Cs
,
23 IN KPROCESSOR_MODE Mode
)
26 // Check if we're in kernel-mode, and force CPL 0 if so.
27 // Otherwise, force CPL 3.
29 return ((Mode
== KernelMode
) ?
30 (Cs
& (0xFFFF & ~RPL_MASK
)) :
31 (RPL_MASK
| (Cs
& 0xFFFF)));
39 Ke386SanitizeFlags(IN ULONG Eflags
,
40 IN KPROCESSOR_MODE Mode
)
43 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
44 // Otherwise, also force interrupt mask on.
46 return ((Mode
== KernelMode
) ?
47 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
48 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
52 // Gets a DR register from a CONTEXT structure
56 KiDrFromContext(IN ULONG Dr
,
59 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
63 // Gets a DR register from a KTRAP_FRAME structure
67 KiDrFromTrapFrame(IN ULONG Dr
,
68 IN PKTRAP_FRAME TrapFrame
)
70 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
78 Ke386SanitizeDr(IN PVOID DrAddress
,
79 IN KPROCESSOR_MODE Mode
)
82 // Check if we're in kernel-mode, and return the address directly if so.
83 // Otherwise, make sure it's not inside the kernel-mode address space.
84 // If it is, then clear the address.
86 return ((Mode
== KernelMode
) ? DrAddress
:
87 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
92 // Enters a Guarded Region
94 #define KeEnterGuardedRegion() \
96 PKTHREAD _Thread = KeGetCurrentThread(); \
99 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
100 ASSERT(_Thread == KeGetCurrentThread()); \
101 ASSERT((_Thread->SpecialApcDisable <= 0) && \
102 (_Thread->SpecialApcDisable != -32768)); \
104 /* Disable Special APCs */ \
105 _Thread->SpecialApcDisable--; \
109 // Leaves a Guarded Region
111 #define KeLeaveGuardedRegion() \
113 PKTHREAD _Thread = KeGetCurrentThread(); \
115 /* Sanity checks */ \
116 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
117 ASSERT(_Thread == KeGetCurrentThread()); \
118 ASSERT(_Thread->SpecialApcDisable < 0); \
120 /* Leave region and check if APCs are OK now */ \
121 if (!(++_Thread->SpecialApcDisable)) \
123 /* Check for Kernel APCs on the list */ \
124 if (!IsListEmpty(&_Thread->ApcState. \
125 ApcListHead[KernelMode])) \
127 /* Check for APC Delivery */ \
128 KiCheckForKernelApcDelivery(); \
134 // TODO: Guarded Mutex Routines
138 // Enters a Critical Region
140 #define KeEnterCriticalRegion() \
142 PKTHREAD _Thread = KeGetCurrentThread(); \
144 /* Sanity checks */ \
145 ASSERT(_Thread == KeGetCurrentThread()); \
146 ASSERT((_Thread->KernelApcDisable <= 0) && \
147 (_Thread->KernelApcDisable != -32768)); \
149 /* Disable Kernel APCs */ \
150 _Thread->KernelApcDisable--; \
154 // Leaves a Critical Region
156 #define KeLeaveCriticalRegion() \
158 PKTHREAD _Thread = KeGetCurrentThread(); \
160 /* Sanity checks */ \
161 ASSERT(_Thread == KeGetCurrentThread()); \
162 ASSERT(_Thread->KernelApcDisable < 0); \
164 /* Enable Kernel APCs */ \
165 _Thread->KernelApcDisable++; \
167 /* Check if Kernel APCs are now enabled */ \
168 if (!(_Thread->KernelApcDisable)) \
170 /* Check if we need to request an APC Delivery */ \
171 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
172 !(_Thread->SpecialApcDisable)) \
174 /* Check for the right environment */ \
175 KiCheckForKernelApcDelivery(); \
182 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
186 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
188 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
189 UNREFERENCED_PARAMETER(SpinLock
);
193 // Spinlock Release at IRQL >= DISPATCH_LEVEL
197 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
199 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
200 UNREFERENCED_PARAMETER(SpinLock
);
204 // This routine protects against multiple CPU acquires, it's meaningless on UP.
208 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
210 UNREFERENCED_PARAMETER(Object
);
214 // This routine protects against multiple CPU acquires, it's meaningless on UP.
218 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
220 UNREFERENCED_PARAMETER(Object
);
225 KiAcquireDispatcherLock(VOID
)
227 /* Raise to DPC level */
228 return KeRaiseIrqlToDpcLevel();
233 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
235 /* Just exit the dispatcher */
236 KiExitDispatcher(OldIrql
);
241 KiAcquireDispatcherLockAtDpcLevel(VOID
)
243 /* This is a no-op at DPC Level for UP systems */
249 KiReleaseDispatcherLockFromDpcLevel(VOID
)
251 /* This is a no-op at DPC Level for UP systems */
256 // This routine makes the thread deferred ready on the boot CPU.
260 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
262 /* Set the thread to deferred state and boot CPU */
263 Thread
->State
= DeferredReady
;
264 Thread
->DeferredProcessor
= 0;
266 /* Make the thread ready immediately */
267 KiDeferredReadyThread(Thread
);
272 KiRescheduleThread(IN BOOLEAN NewThread
,
275 /* This is meaningless on UP systems */
276 UNREFERENCED_PARAMETER(NewThread
);
277 UNREFERENCED_PARAMETER(Cpu
);
281 // This routine protects against multiple CPU acquires, it's meaningless on UP.
285 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
287 UNREFERENCED_PARAMETER(Thread
);
291 // This routine protects against multiple CPU acquires, it's meaningless on UP.
295 KiAcquirePrcbLock(IN PKPRCB Prcb
)
297 UNREFERENCED_PARAMETER(Prcb
);
301 // This routine protects against multiple CPU acquires, it's meaningless on UP.
305 KiReleasePrcbLock(IN PKPRCB Prcb
)
307 UNREFERENCED_PARAMETER(Prcb
);
311 // This routine protects against multiple CPU acquires, it's meaningless on UP.
315 KiAcquireThreadLock(IN PKTHREAD Thread
)
317 UNREFERENCED_PARAMETER(Thread
);
321 // This routine protects against multiple CPU acquires, it's meaningless on UP.
325 KiReleaseThreadLock(IN PKTHREAD Thread
)
327 UNREFERENCED_PARAMETER(Thread
);
331 // This routine protects against multiple CPU acquires, it's meaningless on UP.
335 KiTryThreadLock(IN PKTHREAD Thread
)
337 UNREFERENCED_PARAMETER(Thread
);
343 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
345 /* There are no deferred ready lists on UP systems */
346 UNREFERENCED_PARAMETER(Prcb
);
351 KiRundownThread(IN PKTHREAD Thread
)
354 /* Check if this is the NPX Thread */
355 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
358 KeGetCurrentPrcb()->NpxThread
= NULL
;
366 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
369 /* We deliver instantly on UP */
370 UNREFERENCED_PARAMETER(NeedApc
);
371 UNREFERENCED_PARAMETER(Processor
);
376 KiAcquireTimerLock(IN ULONG Hand
)
378 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
380 /* Nothing to do on UP */
381 UNREFERENCED_PARAMETER(Hand
);
387 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
389 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
391 /* Nothing to do on UP */
392 UNREFERENCED_PARAMETER(LockQueue
);
398 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
402 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
406 /* Try to acquire it */
407 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
409 /* Value changed... wait until it's locked */
410 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
413 /* On debug builds, we use a much slower but useful routine */
414 Kii386SpinOnSpinLock(SpinLock
, 5);
416 /* Otherwise, just yield and keep looping */
424 /* On debug builds, we OR in the KTHREAD */
425 *SpinLock
= KeGetCurrentThread() | 1;
427 /* All is well, break out */
434 // Spinlock Release at IRQL >= DISPATCH_LEVEL
438 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
441 /* Make sure that the threads match */
442 if ((KeGetCurrentThread() | 1) != *SpinLock
)
444 /* They don't, bugcheck */
445 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, SpinLock
, 0, 0, 0);
449 InterlockedAnd(SpinLock
, 0);
454 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
456 LONG OldValue
, NewValue
;
458 /* Make sure we're at a safe level to touch the lock */
459 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
461 /* Start acquire loop */
464 /* Loop until the other CPU releases it */
465 while ((UCHAR
)Object
->Lock
& KOBJECT_LOCK_BIT
)
467 /* Let the CPU know that this is a loop */
471 /* Try acquiring the lock now */
472 NewValue
= InterlockedCompareExchange(&Object
->Lock
,
473 OldValue
| KOBJECT_LOCK_BIT
,
475 } while (NewValue
!= OldValue
);
480 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
482 /* Make sure we're at a safe level to touch the lock */
483 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
486 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
491 KiAcquireDispatcherLock(VOID
)
493 /* Raise to synchronization level and acquire the dispatcher lock */
494 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
499 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
501 /* First release the lock */
502 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
503 LockQueue
[LockQueueDispatcherLock
]);
505 /* Then exit the dispatcher */
506 KiExitDispatcher(OldIrql
);
510 // This routine inserts a thread into the deferred ready list of the given CPU
514 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
516 PKPRCB Prcb
= KeGetCurrentPrcb();
518 /* Set the thread to deferred state and CPU */
519 Thread
->State
= DeferredReady
;
520 Thread
->DeferredProcessor
= Prcb
->Number
;
522 /* Add it on the list */
523 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
528 KiRescheduleThread(IN BOOLEAN NewThread
,
531 /* Check if a new thread needs to be scheduled on a different CPU */
532 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
534 /* Send an IPI to request delivery */
535 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
540 // This routine sets the current thread in a swap busy state, which ensure that
541 // nobody else tries to swap it concurrently.
545 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
547 /* Make sure nobody already set it */
548 ASSERT(Thread
->SwapBusy
== FALSE
);
550 /* Set it ourselves */
551 Thread
->SwapBusy
= TRUE
;
555 // This routine acquires the PRCB lock so that only one caller can touch
556 // volatile PRCB data.
558 // Since this is a simple optimized spin-lock, it must be be only acquired
559 // at dispatcher level or higher!
563 KiAcquirePrcbLock(IN PKPRCB Prcb
)
565 /* Make sure we're at a safe level to touch the PRCB lock */
566 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
568 /* Start acquire loop */
571 /* Acquire the lock and break out if we acquired it first */
572 if (!InterlockedExchange(&Prcb
->PrcbLock
, 1)) break;
574 /* Loop until the other CPU releases it */
577 /* Let the CPU know that this is a loop */
579 } while (Prcb
->PrcbLock
);
584 // This routine releases the PRCB lock so that other callers can touch
585 // volatile PRCB data.
587 // Since this is a simple optimized spin-lock, it must be be only acquired
588 // at dispatcher level or higher!
592 KiReleasePrcbLock(IN PKPRCB Prcb
)
594 /* Make sure it's acquired! */
595 ASSERT(Prcb
->PrcbLock
!= 0);
598 InterlockedAnd(&Prcb
->PrcbLock
, 0);
602 // This routine acquires the thread lock so that only one caller can touch
603 // volatile thread data.
605 // Since this is a simple optimized spin-lock, it must be be only acquired
606 // at dispatcher level or higher!
610 KiAcquireThreadLock(IN PKTHREAD Thread
)
612 /* Make sure we're at a safe level to touch the thread lock */
613 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
615 /* Start acquire loop */
618 /* Acquire the lock and break out if we acquired it first */
619 if (!InterlockedExchange(&Thread
->ThreadLock
, 1)) break;
621 /* Loop until the other CPU releases it */
624 /* Let the CPU know that this is a loop */
626 } while (Thread
->ThreadLock
);
631 // This routine releases the thread lock so that other callers can touch
632 // volatile thread data.
634 // Since this is a simple optimized spin-lock, it must be be only acquired
635 // at dispatcher level or higher!
639 KiReleaseThreadLock(IN PKTHREAD Thread
)
642 InterlockedAnd(&Thread
->ThreadLock
, 0);
647 KiTryThreadLock(IN PKTHREAD Thread
)
651 /* If the lock isn't acquired, return false */
652 if (!Thread
->ThreadLock
) return FALSE
;
654 /* Otherwise, try to acquire it and check the result */
656 Value
= InterlockedExchange(&Thread
->ThreadLock
, &Value
);
658 /* Return the lock state */
659 return (Value
== TRUE
);
664 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
666 /* Scan the deferred ready lists if required */
667 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
672 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
675 /* Check if we need to request APC delivery */
678 /* Check if it's on another CPU */
679 if (KeGetPcr()->Number
!= Cpu
)
681 /* Send an IPI to request delivery */
682 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
686 /* Request a software interrupt */
687 HalRequestSoftwareInterrupt(APC_LEVEL
);
696 KiAcquireApcLock(IN PKTHREAD Thread
,
697 IN PKLOCK_QUEUE_HANDLE Handle
)
699 /* Acquire the lock and raise to synchronization level */
700 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
705 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
706 IN PKLOCK_QUEUE_HANDLE Handle
)
708 /* Acquire the lock */
709 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
714 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
715 IN PKLOCK_QUEUE_HANDLE Handle
)
717 /* Acquire the lock */
718 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
723 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
725 /* Release the lock */
726 KeReleaseInStackQueuedSpinLock(Handle
);
731 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
733 /* Release the lock */
734 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
739 KiAcquireProcessLock(IN PKPROCESS Process
,
740 IN PKLOCK_QUEUE_HANDLE Handle
)
742 /* Acquire the lock and raise to synchronization level */
743 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
748 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
750 /* Release the lock */
751 KeReleaseInStackQueuedSpinLock(Handle
);
756 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
758 /* Release the lock */
759 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
764 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
765 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
767 /* Check if we were called from a threaded DPC */
768 if (KeGetCurrentPrcb()->DpcThreadActive
)
770 /* Lock the Queue, we're not at DPC level */
771 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
775 /* We must be at DPC level, acquire the lock safely */
776 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
777 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
784 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
786 /* Check if we were called from a threaded DPC */
787 if (KeGetCurrentPrcb()->DpcThreadActive
)
789 /* Unlock the Queue, we're not at DPC level */
790 KeReleaseInStackQueuedSpinLock(DeviceLock
);
794 /* We must be at DPC level, release the lock safely */
795 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
796 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
801 // Satisfies the wait of any dispatcher object
803 #define KiSatisfyObjectWait(Object, Thread) \
805 /* Special case for Mutants */ \
806 if ((Object)->Header.Type == MutantObject) \
808 /* Decrease the Signal State */ \
809 (Object)->Header.SignalState--; \
811 /* Check if it's now non-signaled */ \
812 if (!(Object)->Header.SignalState) \
814 /* Set the Owner Thread */ \
815 (Object)->OwnerThread = Thread; \
817 /* Disable APCs if needed */ \
818 Thread->KernelApcDisable = Thread->KernelApcDisable - \
819 (Object)->ApcDisable; \
821 /* Check if it's abandoned */ \
822 if ((Object)->Abandoned) \
825 (Object)->Abandoned = FALSE; \
827 /* Return Status */ \
828 Thread->WaitStatus = STATUS_ABANDONED; \
831 /* Insert it into the Mutant List */ \
832 InsertHeadList(Thread->MutantListHead.Blink, \
833 &(Object)->MutantListEntry); \
836 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
837 EventSynchronizationObject) \
839 /* Synchronization Timers and Events just get un-signaled */ \
840 (Object)->Header.SignalState = 0; \
842 else if ((Object)->Header.Type == SemaphoreObject) \
844 /* These ones can have multiple states, so we only decrease it */ \
845 (Object)->Header.SignalState--; \
850 // Satisfies the wait of a mutant dispatcher object
852 #define KiSatisfyMutantWait(Object, Thread) \
854 /* Decrease the Signal State */ \
855 (Object)->Header.SignalState--; \
857 /* Check if it's now non-signaled */ \
858 if (!(Object)->Header.SignalState) \
860 /* Set the Owner Thread */ \
861 (Object)->OwnerThread = Thread; \
863 /* Disable APCs if needed */ \
864 Thread->KernelApcDisable = Thread->KernelApcDisable - \
865 (Object)->ApcDisable; \
867 /* Check if it's abandoned */ \
868 if ((Object)->Abandoned) \
871 (Object)->Abandoned = FALSE; \
873 /* Return Status */ \
874 Thread->WaitStatus = STATUS_ABANDONED; \
877 /* Insert it into the Mutant List */ \
878 InsertHeadList(Thread->MutantListHead.Blink, \
879 &(Object)->MutantListEntry); \
884 // Satisfies the wait of any nonmutant dispatcher object
886 #define KiSatisfyNonMutantWait(Object) \
888 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
889 EventSynchronizationObject) \
891 /* Synchronization Timers and Events just get un-signaled */ \
892 (Object)->Header.SignalState = 0; \
894 else if ((Object)->Header.Type == SemaphoreObject) \
896 /* These ones can have multiple states, so we only decrease it */ \
897 (Object)->Header.SignalState--; \
902 // Recalculates the due time
906 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
907 IN PLARGE_INTEGER DueTime
,
908 IN OUT PLARGE_INTEGER NewDueTime
)
910 /* Don't do anything for absolute waits */
911 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
913 /* Otherwise, query the interrupt time and recalculate */
914 NewDueTime
->QuadPart
= KeQueryInterruptTime();
915 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
920 // Determines whether a thread should be added to the wait list
924 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
925 IN KPROCESSOR_MODE WaitMode
)
927 /* Check the required conditions */
928 if ((WaitMode
!= KernelMode
) &&
929 (Thread
->EnableStackSwap
) &&
930 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
932 /* We are go for swap */
937 /* Don't swap the thread */
943 // Adds a thread to the wait list
945 #define KiAddThreadToWaitList(Thread, Swappable) \
947 /* Make sure it's swappable */ \
950 /* Insert it into the PRCB's List */ \
951 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
952 &Thread->WaitListEntry); \
957 // Checks if a wait in progress should be interrupted by APCs or an alertable
962 KiCheckAlertability(IN PKTHREAD Thread
,
963 IN BOOLEAN Alertable
,
964 IN KPROCESSOR_MODE WaitMode
)
966 /* Check if the wait is alertable */
969 /* It is, first check if the thread is alerted in this mode */
970 if (Thread
->Alerted
[WaitMode
])
972 /* It is, so bail out of the wait */
973 Thread
->Alerted
[WaitMode
] = FALSE
;
974 return STATUS_ALERTED
;
976 else if ((WaitMode
!= KernelMode
) &&
977 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
979 /* It's isn't, but this is a user wait with queued user APCs */
980 Thread
->ApcState
.UserApcPending
= TRUE
;
981 return STATUS_USER_APC
;
983 else if (Thread
->Alerted
[KernelMode
])
985 /* It isn't that either, but we're alered in kernel mode */
986 Thread
->Alerted
[KernelMode
] = FALSE
;
987 return STATUS_ALERTED
;
990 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
992 /* Not alertable, but this is a user wait with pending user APCs */
993 return STATUS_USER_APC
;
996 /* Otherwise, we're fine */
997 return STATUS_WAIT_0
;
1001 // Called by Wait and Queue code to insert a timer for dispatching.
1002 // Also called by KeSetTimerEx to insert a timer from the caller.
1006 KxInsertTimer(IN PKTIMER Timer
,
1009 PKSPIN_LOCK_QUEUE LockQueue
;
1011 /* Acquire the lock and release the dispatcher lock */
1012 LockQueue
= KiAcquireTimerLock(Hand
);
1013 KiReleaseDispatcherLockFromDpcLevel();
1015 /* Try to insert the timer */
1016 if (KiInsertTimerTable(Timer
, Hand
))
1019 KiCompleteTimer(Timer
, LockQueue
);
1023 /* Do nothing, just release the lock */
1024 KiReleaseTimerLock(LockQueue
);
1029 // Called from Unlink and Queue Insert Code.
1030 // Also called by timer code when canceling an inserted timer.
1031 // Removes a timer from it's tree.
1035 KxRemoveTreeTimer(IN PKTIMER Timer
)
1037 ULONG Hand
= Timer
->Header
.Hand
;
1038 PKSPIN_LOCK_QUEUE LockQueue
;
1039 PKTIMER_TABLE_ENTRY TimerEntry
;
1041 /* Acquire timer lock */
1042 LockQueue
= KiAcquireTimerLock(Hand
);
1044 /* Set the timer as non-inserted */
1045 Timer
->Header
.Inserted
= FALSE
;
1047 /* Remove it from the timer list */
1048 if (RemoveEntryList(&Timer
->TimerListEntry
))
1050 /* Get the entry and check if it's empty */
1051 TimerEntry
= &KiTimerTableListHead
[Hand
];
1052 if (IsListEmpty(&TimerEntry
->Entry
))
1054 /* Clear the time then */
1055 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1059 /* Release the timer lock */
1060 KiReleaseTimerLock(LockQueue
);
1065 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1066 IN LARGE_INTEGER Interval
,
1070 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1072 /* Check the timer's interval to see if it's absolute */
1073 Timer
->Header
.Absolute
= FALSE
;
1074 if (Interval
.HighPart
>= 0)
1076 /* Get the system time and calculate the relative time */
1077 KeQuerySystemTime(&SystemTime
);
1078 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1079 Timer
->Header
.Absolute
= TRUE
;
1081 /* Check if we've already expired */
1082 if (TimeDifference
.HighPart
>= 0)
1084 /* Reset everything */
1085 Timer
->DueTime
.QuadPart
= 0;
1087 Timer
->Header
.Hand
= 0;
1092 /* Update the interval */
1093 Interval
= TimeDifference
;
1097 /* Calculate the due time */
1098 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1099 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1100 Timer
->DueTime
.QuadPart
= DueTime
;
1102 /* Calculate the timer handle */
1103 *Hand
= KiComputeTimerTableIndex(DueTime
);
1104 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1107 #define KxDelayThreadWait() \
1109 /* Setup the Wait Block */ \
1110 Thread->WaitBlockList = TimerBlock; \
1112 /* Setup the timer */ \
1113 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1115 /* Save the due time for the caller */ \
1116 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1118 /* Link the timer to this Wait Block */ \
1119 TimerBlock->NextWaitBlock = TimerBlock; \
1120 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1121 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1123 /* Clear wait status */ \
1124 Thread->WaitStatus = STATUS_SUCCESS; \
1126 /* Setup wait fields */ \
1127 Thread->Alertable = Alertable; \
1128 Thread->WaitReason = DelayExecution; \
1129 Thread->WaitMode = WaitMode; \
1131 /* Check if we can swap the thread's stack */ \
1132 Thread->WaitListEntry.Flink = NULL; \
1133 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1135 /* Set the wait time */ \
1136 Thread->WaitTime = KeTickCount.LowPart;
1138 #define KxMultiThreadWait() \
1139 /* Link wait block array to the thread */ \
1140 Thread->WaitBlockList = WaitBlockArray; \
1142 /* Reset the index */ \
1145 /* Loop wait blocks */ \
1148 /* Fill out the wait block */ \
1149 WaitBlock = &WaitBlockArray[Index]; \
1150 WaitBlock->Object = Object[Index]; \
1151 WaitBlock->WaitKey = (USHORT)Index; \
1152 WaitBlock->WaitType = WaitType; \
1153 WaitBlock->Thread = Thread; \
1155 /* Link to next block */ \
1156 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1158 } while (Index < Count); \
1160 /* Link the last block */ \
1161 WaitBlock->NextWaitBlock = WaitBlockArray; \
1163 /* Set default wait status */ \
1164 Thread->WaitStatus = STATUS_WAIT_0; \
1166 /* Check if we have a timer */ \
1169 /* Link to the block */ \
1170 TimerBlock->NextWaitBlock = WaitBlockArray; \
1172 /* Setup the timer */ \
1173 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1175 /* Save the due time for the caller */ \
1176 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1178 /* Initialize the list */ \
1179 InitializeListHead(&Timer->Header.WaitListHead); \
1182 /* Set wait settings */ \
1183 Thread->Alertable = Alertable; \
1184 Thread->WaitMode = WaitMode; \
1185 Thread->WaitReason = WaitReason; \
1187 /* Check if we can swap the thread's stack */ \
1188 Thread->WaitListEntry.Flink = NULL; \
1189 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1191 /* Set the wait time */ \
1192 Thread->WaitTime = KeTickCount.LowPart;
1194 #define KxSingleThreadWait() \
1195 /* Setup the Wait Block */ \
1196 Thread->WaitBlockList = WaitBlock; \
1197 WaitBlock->WaitKey = STATUS_SUCCESS; \
1198 WaitBlock->Object = Object; \
1199 WaitBlock->WaitType = WaitAny; \
1201 /* Clear wait status */ \
1202 Thread->WaitStatus = STATUS_SUCCESS; \
1204 /* Check if we have a timer */ \
1207 /* Setup the timer */ \
1208 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1210 /* Save the due time for the caller */ \
1211 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1213 /* Pointer to timer block */ \
1214 WaitBlock->NextWaitBlock = TimerBlock; \
1215 TimerBlock->NextWaitBlock = WaitBlock; \
1217 /* Link the timer to this Wait Block */ \
1218 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1219 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1223 /* No timer block, just ourselves */ \
1224 WaitBlock->NextWaitBlock = WaitBlock; \
1227 /* Set wait settings */ \
1228 Thread->Alertable = Alertable; \
1229 Thread->WaitMode = WaitMode; \
1230 Thread->WaitReason = WaitReason; \
1232 /* Check if we can swap the thread's stack */ \
1233 Thread->WaitListEntry.Flink = NULL; \
1234 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1236 /* Set the wait time */ \
1237 Thread->WaitTime = KeTickCount.LowPart;
1239 #define KxQueueThreadWait() \
1240 /* Setup the Wait Block */ \
1241 Thread->WaitBlockList = WaitBlock; \
1242 WaitBlock->WaitKey = STATUS_SUCCESS; \
1243 WaitBlock->Object = Queue; \
1244 WaitBlock->WaitType = WaitAny; \
1245 WaitBlock->Thread = Thread; \
1247 /* Clear wait status */ \
1248 Thread->WaitStatus = STATUS_SUCCESS; \
1250 /* Check if we have a timer */ \
1253 /* Setup the timer */ \
1254 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1256 /* Save the due time for the caller */ \
1257 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1259 /* Pointer to timer block */ \
1260 WaitBlock->NextWaitBlock = TimerBlock; \
1261 TimerBlock->NextWaitBlock = WaitBlock; \
1263 /* Link the timer to this Wait Block */ \
1264 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1265 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1269 /* No timer block, just ourselves */ \
1270 WaitBlock->NextWaitBlock = WaitBlock; \
1273 /* Set wait settings */ \
1274 Thread->Alertable = FALSE; \
1275 Thread->WaitMode = WaitMode; \
1276 Thread->WaitReason = WrQueue; \
1278 /* Check if we can swap the thread's stack */ \
1279 Thread->WaitListEntry.Flink = NULL; \
1280 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1282 /* Set the wait time */ \
1283 Thread->WaitTime = KeTickCount.LowPart;
1290 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1291 IN KPRIORITY Increment
)
1293 PLIST_ENTRY WaitEntry
, WaitList
;
1294 PKWAIT_BLOCK WaitBlock
;
1295 PKTHREAD WaitThread
;
1298 /* Loop the Wait Entries */
1299 WaitList
= &Object
->WaitListHead
;
1300 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1301 WaitEntry
= WaitList
->Flink
;
1304 /* Get the current wait block */
1305 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1307 /* Get the waiting thread */
1308 WaitThread
= WaitBlock
->Thread
;
1310 /* Check the current Wait Mode */
1311 if (WaitBlock
->WaitType
== WaitAny
)
1313 /* Use the actual wait key */
1314 WaitKey
= WaitBlock
->WaitKey
;
1318 /* Otherwise, use STATUS_KERNEL_APC */
1319 WaitKey
= STATUS_KERNEL_APC
;
1322 /* Unwait the thread */
1323 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1326 WaitEntry
= WaitList
->Flink
;
1327 } while (WaitEntry
!= WaitList
);
1331 // Unwaits a Thread waiting on an event
1335 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1336 IN KPRIORITY Increment
)
1338 PLIST_ENTRY WaitEntry
, WaitList
;
1339 PKWAIT_BLOCK WaitBlock
;
1340 PKTHREAD WaitThread
;
1342 /* Loop the Wait Entries */
1343 WaitList
= &Event
->Header
.WaitListHead
;
1344 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1345 WaitEntry
= WaitList
->Flink
;
1348 /* Get the current wait block */
1349 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1351 /* Get the waiting thread */
1352 WaitThread
= WaitBlock
->Thread
;
1354 /* Check the current Wait Mode */
1355 if (WaitBlock
->WaitType
== WaitAny
)
1358 Event
->Header
.SignalState
= 0;
1360 /* Un-signal the event and unwait the thread */
1361 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1365 /* Unwait the thread with STATUS_KERNEL_APC */
1366 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1369 WaitEntry
= WaitList
->Flink
;
1370 } while (WaitEntry
!= WaitList
);
1374 // This routine queues a thread that is ready on the PRCB's ready lists.
1375 // If this thread cannot currently run on this CPU, then the thread is
1376 // added to the deferred ready list instead.
1378 // This routine must be entered with the PRCB lock held and it will exit
1379 // with the PRCB lock released!
1383 KxQueueReadyThread(IN PKTHREAD Thread
,
1390 ASSERT(Prcb
== KeGetCurrentPrcb());
1391 ASSERT(Thread
->State
== Running
);
1392 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1394 /* Check if this thread is allowed to run in this CPU */
1396 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1401 /* Set thread ready for execution */
1402 Thread
->State
= Ready
;
1404 /* Save current priority and if someone had pre-empted it */
1405 Priority
= Thread
->Priority
;
1406 Preempted
= Thread
->Preempted
;
1408 /* We're not pre-empting now, and set the wait time */
1409 Thread
->Preempted
= FALSE
;
1410 Thread
->WaitTime
= KeTickCount
.LowPart
;
1413 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1415 /* Insert this thread in the appropriate order */
1416 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1417 &Thread
->WaitListEntry
) :
1418 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1419 &Thread
->WaitListEntry
);
1421 /* Update the ready summary */
1422 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1425 ASSERT(Priority
== Thread
->Priority
);
1427 /* Release the PRCB lock */
1428 KiReleasePrcbLock(Prcb
);
1432 /* Otherwise, prepare this thread to be deferred */
1433 Thread
->State
= DeferredReady
;
1434 Thread
->DeferredProcessor
= Prcb
->Number
;
1436 /* Release the lock and defer scheduling */
1437 KiReleasePrcbLock(Prcb
);
1438 KiDeferredReadyThread(Thread
);
1443 // This routine scans for an appropriate ready thread to select at the
1444 // given priority and for the given CPU.
1448 KiSelectReadyThread(IN KPRIORITY Priority
,
1451 ULONG PrioritySet
, HighPriority
;
1452 PLIST_ENTRY ListEntry
;
1453 PKTHREAD Thread
= NULL
;
1455 /* Save the current mask and get the priority set for the CPU */
1456 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1457 if (!PrioritySet
) goto Quickie
;
1459 /* Get the highest priority possible */
1460 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1461 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1462 HighPriority
+= Priority
;
1464 /* Make sure the list isn't empty at the highest priority */
1465 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1467 /* Get the first thread on the list */
1468 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1469 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1471 /* Make sure this thread is here for a reason */
1472 ASSERT(HighPriority
== Thread
->Priority
);
1473 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1474 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1476 /* Remove it from the list */
1477 if (RemoveEntryList(&Thread
->WaitListEntry
))
1479 /* The list is empty now, reset the ready summary */
1480 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1483 /* Sanity check and return the thread */
1485 ASSERT((Thread
== NULL
) ||
1486 (Thread
->BasePriority
== 0) ||
1487 (Thread
->Priority
!= 0));
1492 // This routine computes the new priority for a thread. It is only valid for
1493 // threads with priorities in the dynamic priority range.
1497 KiComputeNewPriority(IN PKTHREAD Thread
,
1498 IN SCHAR Adjustment
)
1502 /* Priority sanity checks */
1503 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1504 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1505 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1506 TRUE
: (Thread
->PriorityDecrement
== 0));
1508 /* Get the current priority */
1509 Priority
= Thread
->Priority
;
1510 if (Priority
< LOW_REALTIME_PRIORITY
)
1512 /* Decrease priority by the priority decrement */
1513 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1515 /* Don't go out of bounds */
1516 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1518 /* Reset the priority decrement */
1519 Thread
->PriorityDecrement
= 0;
1523 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1525 /* Return the new priority */
1532 KeGetCurrentThread(VOID
)
1535 /* Return the current thread */
1536 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
1537 #elif defined (_M_AMD64)
1538 return (PRKTHREAD
)__readgsqword(FIELD_OFFSET(KIPCR
, Prcb
.CurrentThread
));
1540 PKPRCB Prcb
= KeGetCurrentPrcb();
1541 return Prcb
->CurrentThread
;
1547 KeGetPreviousMode(VOID
)
1549 /* Return the current mode */
1550 return KeGetCurrentThread()->PreviousMode
;
1556 KeFlushProcessTb(VOID
)
1558 /* Flush the TLB by resetting CR3 */
1560 __asm__("sync\n\tisync\n\t");
1563 // We need to implement this!
1565 ASSERTMSG("Need ARM flush routine\n", FALSE
);
1567 __writecr3(__readcr3());