2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Enters a Guarded Region
12 #define KeEnterGuardedRegion() \
14 PKTHREAD Thread = KeGetCurrentThread(); \
17 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
18 ASSERT(Thread == KeGetCurrentThread()); \
19 ASSERT((Thread->SpecialApcDisable <= 0) && \
20 (Thread->SpecialApcDisable != -32768)); \
22 /* Disable Special APCs */ \
23 Thread->SpecialApcDisable--; \
27 // Leaves a Guarded Region
29 #define KeLeaveGuardedRegion() \
31 PKTHREAD Thread = KeGetCurrentThread(); \
34 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
35 ASSERT(Thread == KeGetCurrentThread()); \
36 ASSERT(Thread->SpecialApcDisable < 0); \
38 /* Leave region and check if APCs are OK now */ \
39 if (!(++Thread->SpecialApcDisable)) \
41 /* Check for Kernel APCs on the list */ \
42 if (!IsListEmpty(&Thread->ApcState. \
43 ApcListHead[KernelMode])) \
45 /* Check for APC Delivery */ \
46 KiCheckForKernelApcDelivery(); \
52 // TODO: Guarded Mutex Routines
56 // Enters a Critical Region
58 #define KeEnterCriticalRegion() \
60 PKTHREAD _Thread = KeGetCurrentThread(); \
63 ASSERT(_Thread == KeGetCurrentThread()); \
64 ASSERT((_Thread->KernelApcDisable <= 0) && \
65 (_Thread->KernelApcDisable != -32768)); \
67 /* Disable Kernel APCs */ \
68 _Thread->KernelApcDisable--; \
72 // Leaves a Critical Region
74 #define KeLeaveCriticalRegion() \
76 PKTHREAD _Thread = KeGetCurrentThread(); \
79 ASSERT(_Thread == KeGetCurrentThread()); \
80 ASSERT(_Thread->KernelApcDisable < 0); \
82 /* Enable Kernel APCs */ \
83 _Thread->KernelApcDisable++; \
85 /* Check if Kernel APCs are now enabled */ \
86 if (!(_Thread->KernelApcDisable)) \
88 /* Check if we need to request an APC Delivery */ \
89 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
90 !(_Thread->KernelApcDisable)) \
92 /* Check for the right environment */ \
93 KiCheckForKernelApcDelivery(); \
99 // Satisfies the wait of any dispatcher object
101 #define KiSatisfyObjectWait(Object, Thread) \
103 /* Special case for Mutants */ \
104 if ((Object)->Header.Type == MutantObject) \
106 /* Decrease the Signal State */ \
107 (Object)->Header.SignalState--; \
109 /* Check if it's now non-signaled */ \
110 if (!(Object)->Header.SignalState) \
112 /* Set the Owner Thread */ \
113 (Object)->OwnerThread = Thread; \
115 /* Disable APCs if needed */ \
116 Thread->KernelApcDisable -= (Object)->ApcDisable; \
118 /* Check if it's abandoned */ \
119 if ((Object)->Abandoned) \
122 (Object)->Abandoned = FALSE; \
124 /* Return Status */ \
125 Thread->WaitStatus = STATUS_ABANDONED; \
128 /* Insert it into the Mutant List */ \
129 InsertHeadList(Thread->MutantListHead.Blink, \
130 &(Object)->MutantListEntry); \
133 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
134 EventSynchronizationObject) \
136 /* Synchronization Timers and Events just get un-signaled */ \
137 (Object)->Header.SignalState = 0; \
139 else if ((Object)->Header.Type == SemaphoreObject) \
141 /* These ones can have multiple states, so we only decrease it */ \
142 (Object)->Header.SignalState--; \
147 // Satisfies the wait of a mutant dispatcher object
149 #define KiSatisfyMutantWait(Object, Thread) \
151 /* Decrease the Signal State */ \
152 (Object)->Header.SignalState--; \
154 /* Check if it's now non-signaled */ \
155 if (!(Object)->Header.SignalState) \
157 /* Set the Owner Thread */ \
158 (Object)->OwnerThread = Thread; \
160 /* Disable APCs if needed */ \
161 Thread->KernelApcDisable -= (Object)->ApcDisable; \
163 /* Check if it's abandoned */ \
164 if ((Object)->Abandoned) \
167 (Object)->Abandoned = FALSE; \
169 /* Return Status */ \
170 Thread->WaitStatus = STATUS_ABANDONED; \
173 /* Insert it into the Mutant List */ \
174 InsertHeadList(Thread->MutantListHead.Blink, \
175 &(Object)->MutantListEntry); \
180 // Satisfies the wait of any nonmutant dispatcher object
182 #define KiSatisfyNonMutantWait(Object, Thread) \
184 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
185 EventSynchronizationObject) \
187 /* Synchronization Timers and Events just get un-signaled */ \
188 (Object)->Header.SignalState = 0; \
190 else if ((Object)->Header.Type == SemaphoreObject) \
192 /* These ones can have multiple states, so we only decrease it */ \
193 (Object)->Header.SignalState--; \
198 // Recalculates the due time
202 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
203 IN PLARGE_INTEGER DueTime
,
204 IN OUT PLARGE_INTEGER NewDueTime
)
206 /* Don't do anything for absolute waits */
207 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
209 /* Otherwise, query the interrupt time and recalculate */
210 NewDueTime
->QuadPart
= KeQueryInterruptTime();
211 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
216 // Determines wether a thread should be added to the wait list
220 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
221 IN KPROCESSOR_MODE WaitMode
)
223 /* Check the required conditions */
224 if ((WaitMode
!= KernelMode
) &&
225 (Thread
->EnableStackSwap
) &&
226 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
228 /* We are go for swap */
233 /* Don't swap the thread */
239 // Adds a thread to the wait list
241 #define KiAddThreadToWaitList(Thread, Swappable) \
243 /* Make sure it's swappable */ \
246 /* Insert it into the PRCB's List */ \
247 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
248 &Thread->WaitListEntry); \
253 // Checks if a wait in progress should be interrupted by APCs or an alertable
258 KiCheckAlertability(IN PKTHREAD Thread
,
259 IN BOOLEAN Alertable
,
260 IN KPROCESSOR_MODE WaitMode
)
262 /* Check if the wait is alertable */
265 /* It is, first check if the thread is alerted in this mode */
266 if (Thread
->Alerted
[WaitMode
])
268 /* It is, so bail out of the wait */
269 Thread
->Alerted
[WaitMode
] = FALSE
;
270 return STATUS_ALERTED
;
272 else if ((WaitMode
!= KernelMode
) &&
273 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
275 /* It's isn't, but this is a user wait with queued user APCs */
276 Thread
->ApcState
.UserApcPending
= TRUE
;
277 return STATUS_USER_APC
;
279 else if (Thread
->Alerted
[KernelMode
])
281 /* It isn't that either, but we're alered in kernel mode */
282 Thread
->Alerted
[KernelMode
] = FALSE
;
283 return STATUS_ALERTED
;
286 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
288 /* Not alertable, but this is a user wait with pending user APCs */
289 return STATUS_USER_APC
;
292 /* Otherwise, we're fine */
293 return STATUS_WAIT_0
;
298 KxDelayThreadWait(IN PKTHREAD Thread
,
299 IN BOOLEAN Alertable
,
300 IN KPROCESSOR_MODE WaitMode
)
303 PKWAIT_BLOCK TimerBlock
= &Thread
->WaitBlock
[TIMER_WAIT_BLOCK
];
305 /* Setup the Wait Block */
306 Thread
->WaitBlockList
= TimerBlock
;
307 TimerBlock
->NextWaitBlock
= TimerBlock
;
309 /* Link the timer to this Wait Block */
310 Thread
->Timer
.Header
.WaitListHead
.Flink
= &TimerBlock
->WaitListEntry
;
311 Thread
->Timer
.Header
.WaitListHead
.Blink
= &TimerBlock
->WaitListEntry
;
313 /* Clear wait status */
314 Thread
->WaitStatus
= STATUS_WAIT_0
;
316 /* Setup wait fields */
317 Thread
->Alertable
= Alertable
;
318 Thread
->WaitReason
= DelayExecution
;
319 Thread
->WaitMode
= WaitMode
;
321 /* Check if we can swap the thread's stack */
322 Thread
->WaitListEntry
.Flink
= NULL
;
323 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
325 /* Set the wait time */
326 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
332 KxMultiThreadWait(IN PKTHREAD Thread
,
333 IN PKWAIT_BLOCK WaitBlock
,
334 IN BOOLEAN Alertable
,
335 IN KWAIT_REASON WaitReason
,
336 IN KPROCESSOR_MODE WaitMode
)
339 PKTIMER ThreadTimer
= &Thread
->Timer
;
341 /* Set default wait status */
342 Thread
->WaitStatus
= STATUS_WAIT_0
;
344 /* Link wait block array to the thread */
345 Thread
->WaitBlockList
= WaitBlock
;
347 /* Initialize the timer list */
348 InitializeListHead(&ThreadTimer
->Header
.WaitListHead
);
350 /* Set wait settings */
351 Thread
->Alertable
= Alertable
;
352 Thread
->WaitMode
= WaitMode
;
353 Thread
->WaitReason
= WaitReason
;
355 /* Check if we can swap the thread's stack */
356 Thread
->WaitListEntry
.Flink
= NULL
;
357 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
359 /* Set the wait time */
360 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
366 KxSingleThreadWait(IN PKTHREAD Thread
,
367 IN PKWAIT_BLOCK WaitBlock
,
369 IN PLARGE_INTEGER Timeout
,
370 IN BOOLEAN Alertable
,
371 IN KWAIT_REASON WaitReason
,
372 IN KPROCESSOR_MODE WaitMode
)
375 PKWAIT_BLOCK TimerBlock
= &Thread
->WaitBlock
[TIMER_WAIT_BLOCK
];
377 /* Setup the Wait Block */
378 Thread
->WaitBlockList
= WaitBlock
;
379 WaitBlock
->WaitKey
= STATUS_WAIT_0
;
380 WaitBlock
->Object
= Object
;
381 WaitBlock
->WaitType
= WaitAny
;
383 /* Clear wait status */
384 Thread
->WaitStatus
= STATUS_WAIT_0
;
386 /* Check if we have a timer */
389 /* Pointer to timer block */
390 WaitBlock
->NextWaitBlock
= TimerBlock
;
391 TimerBlock
->NextWaitBlock
= WaitBlock
;
393 /* Link the timer to this Wait Block */
394 Thread
->Timer
.Header
.WaitListHead
.Flink
= &TimerBlock
->WaitListEntry
;
395 Thread
->Timer
.Header
.WaitListHead
.Blink
= &TimerBlock
->WaitListEntry
;
399 /* No timer block, just ourselves */
400 WaitBlock
->NextWaitBlock
= WaitBlock
;
403 /* Setup wait fields */
404 Thread
->Alertable
= Alertable
;
405 Thread
->WaitReason
= WaitReason
;
406 Thread
->WaitMode
= WaitMode
;
408 /* Check if we can swap the thread's stack */
409 Thread
->WaitListEntry
.Flink
= NULL
;
410 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
412 /* Set the wait time */
413 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
422 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
423 IN KPRIORITY Increment
)
425 PLIST_ENTRY WaitEntry
, WaitList
;
426 PKWAIT_BLOCK CurrentWaitBlock
;
430 /* Loop the Wait Entries */
431 WaitList
= &Object
->WaitListHead
;
432 WaitEntry
= WaitList
->Flink
;
435 /* Get the current wait block */
436 CurrentWaitBlock
= CONTAINING_RECORD(WaitEntry
,
440 /* Get the waiting thread */
441 WaitThread
= CurrentWaitBlock
->Thread
;
443 /* Check the current Wait Mode */
444 if (CurrentWaitBlock
->WaitType
== WaitAny
)
446 /* Use the actual wait key */
447 WaitKey
= CurrentWaitBlock
->WaitKey
;
451 /* Otherwise, use STATUS_KERNEL_APC */
452 WaitKey
= STATUS_KERNEL_APC
;
455 /* Unwait the thread */
456 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
459 WaitEntry
= WaitList
->Flink
;
460 } while (WaitEntry
!= WaitList
);
464 // Unwaits a Thread waiting on an event
468 KxUnwaitThreadForEvent(IN PKEVENT Event
,
469 IN KPRIORITY Increment
)
471 PLIST_ENTRY WaitEntry
, WaitList
;
472 PKWAIT_BLOCK CurrentWaitBlock
;
475 /* Loop the Wait Entries */
476 WaitList
= &Event
->Header
.WaitListHead
;
477 WaitEntry
= WaitList
->Flink
;
480 /* Get the current wait block */
481 CurrentWaitBlock
= CONTAINING_RECORD(WaitEntry
,
485 /* Get the waiting thread */
486 WaitThread
= CurrentWaitBlock
->Thread
;
488 /* Check the current Wait Mode */
489 if (CurrentWaitBlock
->WaitType
== WaitAny
)
492 Event
->Header
.SignalState
= 0;
494 /* Un-signal the event and unwait the thread */
495 KiUnwaitThread(WaitThread
, CurrentWaitBlock
->WaitKey
, Increment
);
500 /* Unwait the thread with STATUS_KERNEL_APC */
501 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
505 WaitEntry
= WaitList
->Flink
;
506 } while (WaitEntry
!= WaitList
);
511 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
515 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
517 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
518 UNREFERENCED_PARAMETER(SpinLock
);
522 // Spinlock Release at IRQL >= DISPATCH_LEVEL
526 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
528 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
529 UNREFERENCED_PARAMETER(SpinLock
);
533 // This routine protects against multiple CPU acquires, it's meaningless on UP.
537 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
539 UNREFERENCED_PARAMETER(Object
);
543 // This routine protects against multiple CPU acquires, it's meaningless on UP.
547 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
549 UNREFERENCED_PARAMETER(Object
);
554 KiAcquireDispatcherLock(VOID
)
556 /* Raise to DPC level */
557 return KeRaiseIrqlToDpcLevel();
562 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
564 /* Just exit the dispatcher */
565 KiExitDispatcher(OldIrql
);
570 KiAcquireDispatcherLockAtDpcLevel(VOID
)
572 /* This is a no-op at DPC Level for UP systems */
578 KiReleaseDispatcherLockFromDpcLevel(VOID
)
580 /* This is a no-op at DPC Level for UP systems */
585 // This routine makes the thread deferred ready on the boot CPU.
589 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
591 /* Set the thread to deferred state and boot CPU */
592 Thread
->State
= DeferredReady
;
593 Thread
->DeferredProcessor
= 0;
595 /* Make the thread ready immediately */
596 KiDeferredReadyThread(Thread
);
601 KiRescheduleThread(IN BOOLEAN NewThread
,
604 /* This is meaningless on UP systems */
605 UNREFERENCED_PARAMETER(NewThread
);
606 UNREFERENCED_PARAMETER(Cpu
);
610 // This routine protects against multiple CPU acquires, it's meaningless on UP.
614 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
616 UNREFERENCED_PARAMETER(Thread
);
620 // This routine protects against multiple CPU acquires, it's meaningless on UP.
624 KiAcquirePrcbLock(IN PKPRCB Prcb
)
626 UNREFERENCED_PARAMETER(Prcb
);
630 // This routine protects against multiple CPU acquires, it's meaningless on UP.
634 KiReleasePrcbLock(IN PKPRCB Prcb
)
636 UNREFERENCED_PARAMETER(Prcb
);
640 // This routine protects against multiple CPU acquires, it's meaningless on UP.
644 KiAcquireThreadLock(IN PKTHREAD Thread
)
646 UNREFERENCED_PARAMETER(Thread
);
650 // This routine protects against multiple CPU acquires, it's meaningless on UP.
654 KiReleaseThreadLock(IN PKTHREAD Thread
)
656 UNREFERENCED_PARAMETER(Thread
);
660 // This routine protects against multiple CPU acquires, it's meaningless on UP.
664 KiTryThreadLock(IN PKTHREAD Thread
)
666 UNREFERENCED_PARAMETER(Thread
);
672 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
674 /* There are no deferred ready lists on UP systems */
675 UNREFERENCED_PARAMETER(Prcb
);
680 KiRundownThread(IN PKTHREAD Thread
)
682 /* Check if this is the NPX Thread */
683 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
686 KeGetCurrentPrcb()->NpxThread
= NULL
;
688 __asm__("fninit\n\t");
697 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
700 /* We deliver instantly on UP */
701 UNREFERENCED_PARAMETER(NeedApc
);
702 UNREFERENCED_PARAMETER(Processor
);
708 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
712 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
716 /* Try to acquire it */
717 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
719 /* Value changed... wait until it's locked */
720 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
723 /* On debug builds, we use a much slower but useful routine */
724 Kii386SpinOnSpinLock(SpinLock
, 5);
726 /* Otherwise, just yield and keep looping */
734 /* On debug builds, we OR in the KTHREAD */
735 *SpinLock
= KeGetCurrentThread() | 1;
737 /* All is well, break out */
744 // Spinlock Release at IRQL >= DISPATCH_LEVEL
748 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
751 /* Make sure that the threads match */
752 if ((KeGetCurrentThread() | 1) != *SpinLock
)
754 /* They don't, bugcheck */
755 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, SpinLock
, 0, 0, 0);
759 InterlockedAnd(SpinLock
, 0);
764 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
766 LONG OldValue
, NewValue
;
768 /* Make sure we're at a safe level to touch the lock */
769 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
771 /* Start acquire loop */
774 /* Loop until the other CPU releases it */
775 while ((UCHAR
)Object
->Lock
& KOBJECT_LOCK_BIT
)
777 /* Let the CPU know that this is a loop */
781 /* Try acquiring the lock now */
782 NewValue
= InterlockedCompareExchange(&Object
->Lock
,
783 OldValue
| KOBJECT_LOCK_BIT
,
785 } while (NewValue
!= OldValue
);
790 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
792 /* Make sure we're at a safe level to touch the lock */
793 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
796 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
801 KiAcquireDispatcherLock(VOID
)
803 /* Raise to synchronization level and acquire the dispatcher lock */
804 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
809 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
811 /* First release the lock */
812 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
813 LockQueue
[LockQueueDispatcherLock
]);
815 /* Then exit the dispatcher */
816 KiExitDispatcher(OldIrql
);
820 // This routine inserts a thread into the deferred ready list of the given CPU
824 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
826 PKPRCB Prcb
= KeGetCurrentPrcb();
828 /* Set the thread to deferred state and CPU */
829 Thread
->State
= DeferredReady
;
830 Thread
->DeferredProcessor
= Prcb
->Number
;
832 /* Add it on the list */
833 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
838 KiRescheduleThread(IN BOOLEAN NewThread
,
841 /* Check if a new thread needs to be scheduled on a different CPU */
842 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
844 /* Send an IPI to request delivery */
845 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
850 // This routine sets the current thread in a swap busy state, which ensure that
851 // nobody else tries to swap it concurrently.
855 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
857 /* Make sure nobody already set it */
858 ASSERT(Thread
->SwapBusy
== FALSE
);
860 /* Set it ourselves */
861 Thread
->SwapBusy
= TRUE
;
865 // This routine acquires the PRCB lock so that only one caller can touch
866 // volatile PRCB data.
868 // Since this is a simple optimized spin-lock, it must be be only acquired
869 // at dispatcher level or higher!
873 KiAcquirePrcbLock(IN PKPRCB Prcb
)
875 /* Make sure we're at a safe level to touch the PRCB lock */
876 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
878 /* Start acquire loop */
881 /* Acquire the lock and break out if we acquired it first */
882 if (!InterlockedExchange(&Prcb
->PrcbLock
, 1)) break;
884 /* Loop until the other CPU releases it */
887 /* Let the CPU know that this is a loop */
889 } while (Prcb
->PrcbLock
);
894 // This routine releases the PRCB lock so that other callers can touch
895 // volatile PRCB data.
897 // Since this is a simple optimized spin-lock, it must be be only acquired
898 // at dispatcher level or higher!
902 KiReleasePrcbLock(IN PKPRCB Prcb
)
904 /* Make sure it's acquired! */
905 ASSERT(Prcb
->PrcbLock
!= 0);
908 InterlockedAnd(&Prcb
->PrcbLock
, 0);
912 // This routine acquires the thread lock so that only one caller can touch
913 // volatile thread data.
915 // Since this is a simple optimized spin-lock, it must be be only acquired
916 // at dispatcher level or higher!
920 KiAcquireThreadLock(IN PKTHREAD Thread
)
922 /* Make sure we're at a safe level to touch the thread lock */
923 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
925 /* Start acquire loop */
928 /* Acquire the lock and break out if we acquired it first */
929 if (!InterlockedExchange(&Thread
->ThreadLock
, 1)) break;
931 /* Loop until the other CPU releases it */
934 /* Let the CPU know that this is a loop */
936 } while (Thread
->ThreadLock
);
941 // This routine releases the thread lock so that other callers can touch
942 // volatile thread data.
944 // Since this is a simple optimized spin-lock, it must be be only acquired
945 // at dispatcher level or higher!
949 KiReleaseThreadLock(IN PKTHREAD Thread
)
952 InterlockedAnd(&Thread
->ThreadLock
, 0);
957 KiTryThreadLock(IN PKTHREAD Thread
)
961 /* If the lock isn't acquired, return false */
962 if (!Thread
->ThreadLock
) return FALSE
;
964 /* Otherwise, try to acquire it and check the result */
966 Value
= InterlockedExchange(&Thread
->ThreadLock
, &Value
);
968 /* Return the lock state */
969 return (Value
== TRUE
);
974 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
976 /* Scan the deferred ready lists if required */
977 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
982 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
985 /* Check if we need to request APC delivery */
988 /* Check if it's on another CPU */
989 if (KeGetPcr()->Number
!= Cpu
)
991 /* Send an IPI to request delivery */
992 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
996 /* Request a software interrupt */
997 HalRequestSoftwareInterrupt(APC_LEVEL
);
1006 KiAcquireApcLock(IN PKTHREAD Thread
,
1007 IN PKLOCK_QUEUE_HANDLE Handle
)
1009 /* Acquire the lock and raise to synchronization level */
1010 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
1015 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
1016 IN PKLOCK_QUEUE_HANDLE Handle
)
1018 /* Acquire the lock */
1019 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
1024 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
1025 IN PKLOCK_QUEUE_HANDLE Handle
)
1027 /* Acquire the lock */
1028 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
1033 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1035 /* Release the lock */
1036 KeReleaseInStackQueuedSpinLock(Handle
);
1041 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1043 /* Release the lock */
1044 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1049 KiAcquireProcessLock(IN PKPROCESS Process
,
1050 IN PKLOCK_QUEUE_HANDLE Handle
)
1052 /* Acquire the lock and raise to synchronization level */
1053 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
1058 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1060 /* Release the lock */
1061 KeReleaseInStackQueuedSpinLock(Handle
);
1066 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1068 /* Release the lock */
1069 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1074 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
1075 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1077 /* Check if we were called from a threaded DPC */
1078 if (KeGetCurrentPrcb()->DpcThreadActive
)
1080 /* Lock the Queue, we're not at DPC level */
1081 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
1085 /* We must be at DPC level, acquire the lock safely */
1086 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1087 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
1094 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1096 /* Check if we were called from a threaded DPC */
1097 if (KeGetCurrentPrcb()->DpcThreadActive
)
1099 /* Unlock the Queue, we're not at DPC level */
1100 KeReleaseInStackQueuedSpinLock(DeviceLock
);
1104 /* We must be at DPC level, release the lock safely */
1105 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1106 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
1111 // This routine queues a thread that is ready on the PRCB's ready lists.
1112 // If this thread cannot currently run on this CPU, then the thread is
1113 // added to the deferred ready list instead.
1115 // This routine must be entered with the PRCB lock held and it will exit
1116 // with the PRCB lock released!
1120 KxQueueReadyThread(IN PKTHREAD Thread
,
1127 ASSERT(Prcb
== KeGetCurrentPrcb());
1128 ASSERT(Thread
->State
== Running
);
1129 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1131 /* Check if this thread is allowed to run in this CPU */
1133 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1138 /* Set thread ready for execution */
1139 Thread
->State
= Ready
;
1141 /* Save current priority and if someone had pre-empted it */
1142 Priority
= Thread
->Priority
;
1143 Preempted
= Thread
->Preempted
;
1145 /* We're not pre-empting now, and set the wait time */
1146 Thread
->Preempted
= FALSE
;
1147 Thread
->WaitTime
= KeTickCount
.LowPart
;
1150 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1152 /* Insert this thread in the appropriate order */
1153 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1154 &Thread
->WaitListEntry
) :
1155 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1156 &Thread
->WaitListEntry
);
1158 /* Update the ready summary */
1159 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1162 ASSERT(Priority
== Thread
->Priority
);
1164 /* Release the PRCB lock */
1165 KiReleasePrcbLock(Prcb
);
1169 /* Otherwise, prepare this thread to be deferred */
1170 Thread
->State
= DeferredReady
;
1171 Thread
->DeferredProcessor
= Prcb
->Number
;
1173 /* Release the lock and defer scheduling */
1174 KiReleasePrcbLock(Prcb
);
1175 KiDeferredReadyThread(Thread
);
1180 // This routine scans for an appropriate ready thread to select at the
1181 // given priority and for the given CPU.
1185 KiSelectReadyThread(IN KPRIORITY Priority
,
1188 LONG PriorityMask
, PrioritySet
, HighPriority
;
1189 PLIST_ENTRY ListEntry
;
1192 /* Save the current mask and get the priority set for the CPU */
1193 PriorityMask
= Priority
;
1194 PrioritySet
= Prcb
->ReadySummary
>> (UCHAR
)Priority
;
1195 if (!PrioritySet
) return NULL
;
1197 /* Get the highest priority possible */
1198 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1199 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1200 HighPriority
+= PriorityMask
;
1202 /* Make sure the list isn't at highest priority */
1203 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1205 /* Get the first thread on the list */
1206 ListEntry
= &Prcb
->DispatcherReadyListHead
[HighPriority
];
1207 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1209 /* Make sure this thread is here for a reason */
1210 ASSERT(HighPriority
== Thread
->Priority
);
1211 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1212 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1214 /* Remove it from the list */
1215 RemoveEntryList(&Thread
->WaitListEntry
);
1216 if (IsListEmpty(&Thread
->WaitListEntry
))
1218 /* The list is empty now, reset the ready summary */
1219 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1222 /* Sanity check and return the thread */
1223 ASSERT((Thread
== NULL
) ||
1224 (Thread
->BasePriority
== 0) ||
1225 (Thread
->Priority
!= 0));
1230 // This routine computes the new priority for a thread. It is only valid for
1231 // threads with priorities in the dynamic priority range.
1235 KiComputeNewPriority(IN PKTHREAD Thread
)
1239 /* Priority sanity checks */
1240 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1241 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1242 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1243 TRUE
: (Thread
->PriorityDecrement
== 0));
1245 /* Get the current priority */
1246 Priority
= Thread
->Priority
;
1247 if (Priority
< LOW_REALTIME_PRIORITY
)
1249 /* Decrease priority by the priority decrement */
1250 Priority
-= (Thread
->PriorityDecrement
+ 1);
1252 /* Don't go out of bounds */
1253 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1255 /* Reset the priority decrement */
1256 Thread
->PriorityDecrement
= 0;
1260 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1262 /* Return the new priority */
1268 KeGetCurrentThread(VOID
)
1270 /* Return the current thread */
1271 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
1276 KeGetPreviousMode(VOID
)
1278 /* Return the current mode */
1279 return KeGetCurrentThread()->PreviousMode
;