2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
17 // Sanitizes a selector
21 Ke386SanitizeSeg(IN ULONG Cs
,
22 IN KPROCESSOR_MODE Mode
)
25 // Check if we're in kernel-mode, and force CPL 0 if so.
26 // Otherwise, force CPL 3.
28 return ((Mode
== KernelMode
) ?
29 (Cs
& (0xFFFF & ~RPL_MASK
)) :
30 (RPL_MASK
| (Cs
& 0xFFFF)));
38 Ke386SanitizeFlags(IN ULONG Eflags
,
39 IN KPROCESSOR_MODE Mode
)
42 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
43 // Otherwise, also force interrupt mask on.
45 return ((Mode
== KernelMode
) ?
46 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
47 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
51 // Gets a DR register from a CONTEXT structure
55 KiDrFromContext(IN ULONG Dr
,
58 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
62 // Gets a DR register from a KTRAP_FRAME structure
66 KiDrFromTrapFrame(IN ULONG Dr
,
67 IN PKTRAP_FRAME TrapFrame
)
69 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
77 Ke386SanitizeDr(IN PVOID DrAddress
,
78 IN KPROCESSOR_MODE Mode
)
81 // Check if we're in kernel-mode, and return the address directly if so.
82 // Otherwise, make sure it's not inside the kernel-mode address space.
83 // If it is, then clear the address.
85 return ((Mode
== KernelMode
) ? DrAddress
:
86 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
90 // Enters a Guarded Region
92 #define KeEnterGuardedRegion() \
94 PKTHREAD _Thread = KeGetCurrentThread(); \
97 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
98 ASSERT(_Thread == KeGetCurrentThread()); \
99 ASSERT((_Thread->SpecialApcDisable <= 0) && \
100 (_Thread->SpecialApcDisable != -32768)); \
102 /* Disable Special APCs */ \
103 _Thread->SpecialApcDisable--; \
107 // Leaves a Guarded Region
109 #define KeLeaveGuardedRegion() \
111 PKTHREAD _Thread = KeGetCurrentThread(); \
113 /* Sanity checks */ \
114 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
115 ASSERT(_Thread == KeGetCurrentThread()); \
116 ASSERT(_Thread->SpecialApcDisable < 0); \
118 /* Leave region and check if APCs are OK now */ \
119 if (!(++_Thread->SpecialApcDisable)) \
121 /* Check for Kernel APCs on the list */ \
122 if (!IsListEmpty(&_Thread->ApcState. \
123 ApcListHead[KernelMode])) \
125 /* Check for APC Delivery */ \
126 KiCheckForKernelApcDelivery(); \
132 // TODO: Guarded Mutex Routines
136 // Enters a Critical Region
138 #define KeEnterCriticalRegion() \
140 PKTHREAD _Thread = KeGetCurrentThread(); \
142 /* Sanity checks */ \
143 ASSERT(_Thread == KeGetCurrentThread()); \
144 ASSERT((_Thread->KernelApcDisable <= 0) && \
145 (_Thread->KernelApcDisable != -32768)); \
147 /* Disable Kernel APCs */ \
148 _Thread->KernelApcDisable--; \
152 // Leaves a Critical Region
154 #define KeLeaveCriticalRegion() \
156 PKTHREAD _Thread = KeGetCurrentThread(); \
158 /* Sanity checks */ \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->KernelApcDisable < 0); \
162 /* Enable Kernel APCs */ \
163 _Thread->KernelApcDisable++; \
165 /* Check if Kernel APCs are now enabled */ \
166 if (!(_Thread->KernelApcDisable)) \
168 /* Check if we need to request an APC Delivery */ \
169 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
170 !(_Thread->SpecialApcDisable)) \
172 /* Check for the right environment */ \
173 KiCheckForKernelApcDelivery(); \
179 // Satisfies the wait of any dispatcher object
181 #define KiSatisfyObjectWait(Object, Thread) \
183 /* Special case for Mutants */ \
184 if ((Object)->Header.Type == MutantObject) \
186 /* Decrease the Signal State */ \
187 (Object)->Header.SignalState--; \
189 /* Check if it's now non-signaled */ \
190 if (!(Object)->Header.SignalState) \
192 /* Set the Owner Thread */ \
193 (Object)->OwnerThread = Thread; \
195 /* Disable APCs if needed */ \
196 Thread->KernelApcDisable -= (Object)->ApcDisable; \
198 /* Check if it's abandoned */ \
199 if ((Object)->Abandoned) \
202 (Object)->Abandoned = FALSE; \
204 /* Return Status */ \
205 Thread->WaitStatus = STATUS_ABANDONED; \
208 /* Insert it into the Mutant List */ \
209 InsertHeadList(Thread->MutantListHead.Blink, \
210 &(Object)->MutantListEntry); \
213 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
214 EventSynchronizationObject) \
216 /* Synchronization Timers and Events just get un-signaled */ \
217 (Object)->Header.SignalState = 0; \
219 else if ((Object)->Header.Type == SemaphoreObject) \
221 /* These ones can have multiple states, so we only decrease it */ \
222 (Object)->Header.SignalState--; \
227 // Satisfies the wait of a mutant dispatcher object
229 #define KiSatisfyMutantWait(Object, Thread) \
231 /* Decrease the Signal State */ \
232 (Object)->Header.SignalState--; \
234 /* Check if it's now non-signaled */ \
235 if (!(Object)->Header.SignalState) \
237 /* Set the Owner Thread */ \
238 (Object)->OwnerThread = Thread; \
240 /* Disable APCs if needed */ \
241 Thread->KernelApcDisable -= (Object)->ApcDisable; \
243 /* Check if it's abandoned */ \
244 if ((Object)->Abandoned) \
247 (Object)->Abandoned = FALSE; \
249 /* Return Status */ \
250 Thread->WaitStatus = STATUS_ABANDONED; \
253 /* Insert it into the Mutant List */ \
254 InsertHeadList(Thread->MutantListHead.Blink, \
255 &(Object)->MutantListEntry); \
260 // Satisfies the wait of any nonmutant dispatcher object
262 #define KiSatisfyNonMutantWait(Object, Thread) \
264 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
265 EventSynchronizationObject) \
267 /* Synchronization Timers and Events just get un-signaled */ \
268 (Object)->Header.SignalState = 0; \
270 else if ((Object)->Header.Type == SemaphoreObject) \
272 /* These ones can have multiple states, so we only decrease it */ \
273 (Object)->Header.SignalState--; \
278 // Recalculates the due time
282 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
283 IN PLARGE_INTEGER DueTime
,
284 IN OUT PLARGE_INTEGER NewDueTime
)
286 /* Don't do anything for absolute waits */
287 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
289 /* Otherwise, query the interrupt time and recalculate */
290 NewDueTime
->QuadPart
= KeQueryInterruptTime();
291 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
296 // Determines wether a thread should be added to the wait list
300 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
301 IN KPROCESSOR_MODE WaitMode
)
303 /* Check the required conditions */
304 if ((WaitMode
!= KernelMode
) &&
305 (Thread
->EnableStackSwap
) &&
306 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
308 /* We are go for swap */
313 /* Don't swap the thread */
319 // Adds a thread to the wait list
321 #define KiAddThreadToWaitList(Thread, Swappable) \
323 /* Make sure it's swappable */ \
326 /* Insert it into the PRCB's List */ \
327 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
328 &Thread->WaitListEntry); \
333 // Checks if a wait in progress should be interrupted by APCs or an alertable
338 KiCheckAlertability(IN PKTHREAD Thread
,
339 IN BOOLEAN Alertable
,
340 IN KPROCESSOR_MODE WaitMode
)
342 /* Check if the wait is alertable */
345 /* It is, first check if the thread is alerted in this mode */
346 if (Thread
->Alerted
[WaitMode
])
348 /* It is, so bail out of the wait */
349 Thread
->Alerted
[WaitMode
] = FALSE
;
350 return STATUS_ALERTED
;
352 else if ((WaitMode
!= KernelMode
) &&
353 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
355 /* It's isn't, but this is a user wait with queued user APCs */
356 Thread
->ApcState
.UserApcPending
= TRUE
;
357 return STATUS_USER_APC
;
359 else if (Thread
->Alerted
[KernelMode
])
361 /* It isn't that either, but we're alered in kernel mode */
362 Thread
->Alerted
[KernelMode
] = FALSE
;
363 return STATUS_ALERTED
;
366 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
368 /* Not alertable, but this is a user wait with pending user APCs */
369 return STATUS_USER_APC
;
372 /* Otherwise, we're fine */
373 return STATUS_WAIT_0
;
378 KxDelayThreadWait(IN PKTHREAD Thread
,
379 IN BOOLEAN Alertable
,
380 IN KPROCESSOR_MODE WaitMode
)
383 PKWAIT_BLOCK TimerBlock
= &Thread
->WaitBlock
[TIMER_WAIT_BLOCK
];
385 /* Setup the Wait Block */
386 Thread
->WaitBlockList
= TimerBlock
;
387 TimerBlock
->NextWaitBlock
= TimerBlock
;
389 /* Link the timer to this Wait Block */
390 Thread
->Timer
.Header
.WaitListHead
.Flink
= &TimerBlock
->WaitListEntry
;
391 Thread
->Timer
.Header
.WaitListHead
.Blink
= &TimerBlock
->WaitListEntry
;
393 /* Clear wait status */
394 Thread
->WaitStatus
= STATUS_WAIT_0
;
396 /* Setup wait fields */
397 Thread
->Alertable
= Alertable
;
398 Thread
->WaitReason
= DelayExecution
;
399 Thread
->WaitMode
= WaitMode
;
401 /* Check if we can swap the thread's stack */
402 Thread
->WaitListEntry
.Flink
= NULL
;
403 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
405 /* Set the wait time */
406 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
412 KxMultiThreadWait(IN PKTHREAD Thread
,
413 IN PKWAIT_BLOCK WaitBlock
,
414 IN BOOLEAN Alertable
,
415 IN KWAIT_REASON WaitReason
,
416 IN KPROCESSOR_MODE WaitMode
)
419 PKTIMER ThreadTimer
= &Thread
->Timer
;
421 /* Set default wait status */
422 Thread
->WaitStatus
= STATUS_WAIT_0
;
424 /* Link wait block array to the thread */
425 Thread
->WaitBlockList
= WaitBlock
;
427 /* Initialize the timer list */
428 InitializeListHead(&ThreadTimer
->Header
.WaitListHead
);
430 /* Set wait settings */
431 Thread
->Alertable
= Alertable
;
432 Thread
->WaitMode
= WaitMode
;
433 Thread
->WaitReason
= WaitReason
;
435 /* Check if we can swap the thread's stack */
436 Thread
->WaitListEntry
.Flink
= NULL
;
437 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
439 /* Set the wait time */
440 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
446 KxSingleThreadWait(IN PKTHREAD Thread
,
447 IN PKWAIT_BLOCK WaitBlock
,
449 IN PLARGE_INTEGER Timeout
,
450 IN BOOLEAN Alertable
,
451 IN KWAIT_REASON WaitReason
,
452 IN KPROCESSOR_MODE WaitMode
)
455 PKWAIT_BLOCK TimerBlock
= &Thread
->WaitBlock
[TIMER_WAIT_BLOCK
];
457 /* Setup the Wait Block */
458 Thread
->WaitBlockList
= WaitBlock
;
459 WaitBlock
->WaitKey
= STATUS_WAIT_0
;
460 WaitBlock
->Object
= Object
;
461 WaitBlock
->WaitType
= WaitAny
;
463 /* Clear wait status */
464 Thread
->WaitStatus
= STATUS_WAIT_0
;
466 /* Check if we have a timer */
469 /* Pointer to timer block */
470 WaitBlock
->NextWaitBlock
= TimerBlock
;
471 TimerBlock
->NextWaitBlock
= WaitBlock
;
473 /* Link the timer to this Wait Block */
474 Thread
->Timer
.Header
.WaitListHead
.Flink
= &TimerBlock
->WaitListEntry
;
475 Thread
->Timer
.Header
.WaitListHead
.Blink
= &TimerBlock
->WaitListEntry
;
479 /* No timer block, just ourselves */
480 WaitBlock
->NextWaitBlock
= WaitBlock
;
483 /* Setup wait fields */
484 Thread
->Alertable
= Alertable
;
485 Thread
->WaitReason
= WaitReason
;
486 Thread
->WaitMode
= WaitMode
;
488 /* Check if we can swap the thread's stack */
489 Thread
->WaitListEntry
.Flink
= NULL
;
490 Swappable
= KiCheckThreadStackSwap(Thread
, WaitMode
);
492 /* Set the wait time */
493 Thread
->WaitTime
= ((PLARGE_INTEGER
)&KeTickCount
)->LowPart
;
502 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
503 IN KPRIORITY Increment
)
505 PLIST_ENTRY WaitEntry
, WaitList
;
506 PKWAIT_BLOCK WaitBlock
;
510 /* Loop the Wait Entries */
511 WaitList
= &Object
->WaitListHead
;
512 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
513 WaitEntry
= WaitList
->Flink
;
516 /* Get the current wait block */
517 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
519 /* Get the waiting thread */
520 WaitThread
= WaitBlock
->Thread
;
522 /* Check the current Wait Mode */
523 if (WaitBlock
->WaitType
== WaitAny
)
525 /* Use the actual wait key */
526 WaitKey
= WaitBlock
->WaitKey
;
530 /* Otherwise, use STATUS_KERNEL_APC */
531 WaitKey
= STATUS_KERNEL_APC
;
534 /* Unwait the thread */
535 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
538 WaitEntry
= WaitList
->Flink
;
539 } while (WaitEntry
!= WaitList
);
543 // Unwaits a Thread waiting on an event
547 KxUnwaitThreadForEvent(IN PKEVENT Event
,
548 IN KPRIORITY Increment
)
550 PLIST_ENTRY WaitEntry
, WaitList
;
551 PKWAIT_BLOCK WaitBlock
;
554 /* Loop the Wait Entries */
555 WaitList
= &Event
->Header
.WaitListHead
;
556 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
557 WaitEntry
= WaitList
->Flink
;
560 /* Get the current wait block */
561 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
563 /* Get the waiting thread */
564 WaitThread
= WaitBlock
->Thread
;
566 /* Check the current Wait Mode */
567 if (WaitBlock
->WaitType
== WaitAny
)
570 Event
->Header
.SignalState
= 0;
572 /* Un-signal the event and unwait the thread */
573 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
577 /* Unwait the thread with STATUS_KERNEL_APC */
578 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
581 WaitEntry
= WaitList
->Flink
;
582 } while (WaitEntry
!= WaitList
);
587 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
591 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
593 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
594 UNREFERENCED_PARAMETER(SpinLock
);
598 // Spinlock Release at IRQL >= DISPATCH_LEVEL
602 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
604 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
605 UNREFERENCED_PARAMETER(SpinLock
);
609 // This routine protects against multiple CPU acquires, it's meaningless on UP.
613 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
615 UNREFERENCED_PARAMETER(Object
);
619 // This routine protects against multiple CPU acquires, it's meaningless on UP.
623 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
625 UNREFERENCED_PARAMETER(Object
);
630 KiAcquireDispatcherLock(VOID
)
632 /* Raise to DPC level */
633 return KeRaiseIrqlToDpcLevel();
638 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
640 /* Just exit the dispatcher */
641 KiExitDispatcher(OldIrql
);
646 KiAcquireDispatcherLockAtDpcLevel(VOID
)
648 /* This is a no-op at DPC Level for UP systems */
654 KiReleaseDispatcherLockFromDpcLevel(VOID
)
656 /* This is a no-op at DPC Level for UP systems */
661 // This routine makes the thread deferred ready on the boot CPU.
665 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
667 /* Set the thread to deferred state and boot CPU */
668 Thread
->State
= DeferredReady
;
669 Thread
->DeferredProcessor
= 0;
671 /* Make the thread ready immediately */
672 KiDeferredReadyThread(Thread
);
677 KiRescheduleThread(IN BOOLEAN NewThread
,
680 /* This is meaningless on UP systems */
681 UNREFERENCED_PARAMETER(NewThread
);
682 UNREFERENCED_PARAMETER(Cpu
);
686 // This routine protects against multiple CPU acquires, it's meaningless on UP.
690 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
692 UNREFERENCED_PARAMETER(Thread
);
696 // This routine protects against multiple CPU acquires, it's meaningless on UP.
700 KiAcquirePrcbLock(IN PKPRCB Prcb
)
702 UNREFERENCED_PARAMETER(Prcb
);
706 // This routine protects against multiple CPU acquires, it's meaningless on UP.
710 KiReleasePrcbLock(IN PKPRCB Prcb
)
712 UNREFERENCED_PARAMETER(Prcb
);
716 // This routine protects against multiple CPU acquires, it's meaningless on UP.
720 KiAcquireThreadLock(IN PKTHREAD Thread
)
722 UNREFERENCED_PARAMETER(Thread
);
726 // This routine protects against multiple CPU acquires, it's meaningless on UP.
730 KiReleaseThreadLock(IN PKTHREAD Thread
)
732 UNREFERENCED_PARAMETER(Thread
);
736 // This routine protects against multiple CPU acquires, it's meaningless on UP.
740 KiTryThreadLock(IN PKTHREAD Thread
)
742 UNREFERENCED_PARAMETER(Thread
);
748 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
750 /* There are no deferred ready lists on UP systems */
751 UNREFERENCED_PARAMETER(Prcb
);
756 KiRundownThread(IN PKTHREAD Thread
)
758 /* Check if this is the NPX Thread */
759 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
762 KeGetCurrentPrcb()->NpxThread
= NULL
;
769 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
772 /* We deliver instantly on UP */
773 UNREFERENCED_PARAMETER(NeedApc
);
774 UNREFERENCED_PARAMETER(Processor
);
780 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
784 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
788 /* Try to acquire it */
789 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
791 /* Value changed... wait until it's locked */
792 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
795 /* On debug builds, we use a much slower but useful routine */
796 Kii386SpinOnSpinLock(SpinLock
, 5);
798 /* Otherwise, just yield and keep looping */
806 /* On debug builds, we OR in the KTHREAD */
807 *SpinLock
= KeGetCurrentThread() | 1;
809 /* All is well, break out */
816 // Spinlock Release at IRQL >= DISPATCH_LEVEL
820 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
823 /* Make sure that the threads match */
824 if ((KeGetCurrentThread() | 1) != *SpinLock
)
826 /* They don't, bugcheck */
827 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, SpinLock
, 0, 0, 0);
831 InterlockedAnd(SpinLock
, 0);
836 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
838 LONG OldValue
, NewValue
;
840 /* Make sure we're at a safe level to touch the lock */
841 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
843 /* Start acquire loop */
846 /* Loop until the other CPU releases it */
847 while ((UCHAR
)Object
->Lock
& KOBJECT_LOCK_BIT
)
849 /* Let the CPU know that this is a loop */
853 /* Try acquiring the lock now */
854 NewValue
= InterlockedCompareExchange(&Object
->Lock
,
855 OldValue
| KOBJECT_LOCK_BIT
,
857 } while (NewValue
!= OldValue
);
862 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
864 /* Make sure we're at a safe level to touch the lock */
865 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
868 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
873 KiAcquireDispatcherLock(VOID
)
875 /* Raise to synchronization level and acquire the dispatcher lock */
876 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
881 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
883 /* First release the lock */
884 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
885 LockQueue
[LockQueueDispatcherLock
]);
887 /* Then exit the dispatcher */
888 KiExitDispatcher(OldIrql
);
892 // This routine inserts a thread into the deferred ready list of the given CPU
896 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
898 PKPRCB Prcb
= KeGetCurrentPrcb();
900 /* Set the thread to deferred state and CPU */
901 Thread
->State
= DeferredReady
;
902 Thread
->DeferredProcessor
= Prcb
->Number
;
904 /* Add it on the list */
905 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
910 KiRescheduleThread(IN BOOLEAN NewThread
,
913 /* Check if a new thread needs to be scheduled on a different CPU */
914 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
916 /* Send an IPI to request delivery */
917 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
922 // This routine sets the current thread in a swap busy state, which ensure that
923 // nobody else tries to swap it concurrently.
927 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
929 /* Make sure nobody already set it */
930 ASSERT(Thread
->SwapBusy
== FALSE
);
932 /* Set it ourselves */
933 Thread
->SwapBusy
= TRUE
;
937 // This routine acquires the PRCB lock so that only one caller can touch
938 // volatile PRCB data.
940 // Since this is a simple optimized spin-lock, it must be be only acquired
941 // at dispatcher level or higher!
945 KiAcquirePrcbLock(IN PKPRCB Prcb
)
947 /* Make sure we're at a safe level to touch the PRCB lock */
948 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
950 /* Start acquire loop */
953 /* Acquire the lock and break out if we acquired it first */
954 if (!InterlockedExchange(&Prcb
->PrcbLock
, 1)) break;
956 /* Loop until the other CPU releases it */
959 /* Let the CPU know that this is a loop */
961 } while (Prcb
->PrcbLock
);
966 // This routine releases the PRCB lock so that other callers can touch
967 // volatile PRCB data.
969 // Since this is a simple optimized spin-lock, it must be be only acquired
970 // at dispatcher level or higher!
974 KiReleasePrcbLock(IN PKPRCB Prcb
)
976 /* Make sure it's acquired! */
977 ASSERT(Prcb
->PrcbLock
!= 0);
980 InterlockedAnd(&Prcb
->PrcbLock
, 0);
984 // This routine acquires the thread lock so that only one caller can touch
985 // volatile thread data.
987 // Since this is a simple optimized spin-lock, it must be be only acquired
988 // at dispatcher level or higher!
992 KiAcquireThreadLock(IN PKTHREAD Thread
)
994 /* Make sure we're at a safe level to touch the thread lock */
995 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
997 /* Start acquire loop */
1000 /* Acquire the lock and break out if we acquired it first */
1001 if (!InterlockedExchange(&Thread
->ThreadLock
, 1)) break;
1003 /* Loop until the other CPU releases it */
1006 /* Let the CPU know that this is a loop */
1008 } while (Thread
->ThreadLock
);
1013 // This routine releases the thread lock so that other callers can touch
1014 // volatile thread data.
1016 // Since this is a simple optimized spin-lock, it must be be only acquired
1017 // at dispatcher level or higher!
1021 KiReleaseThreadLock(IN PKTHREAD Thread
)
1024 InterlockedAnd(&Thread
->ThreadLock
, 0);
1029 KiTryThreadLock(IN PKTHREAD Thread
)
1033 /* If the lock isn't acquired, return false */
1034 if (!Thread
->ThreadLock
) return FALSE
;
1036 /* Otherwise, try to acquire it and check the result */
1038 Value
= InterlockedExchange(&Thread
->ThreadLock
, &Value
);
1040 /* Return the lock state */
1041 return (Value
== TRUE
);
1046 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
1048 /* Scan the deferred ready lists if required */
1049 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
1054 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
1057 /* Check if we need to request APC delivery */
1060 /* Check if it's on another CPU */
1061 if (KeGetPcr()->Number
!= Cpu
)
1063 /* Send an IPI to request delivery */
1064 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
1068 /* Request a software interrupt */
1069 HalRequestSoftwareInterrupt(APC_LEVEL
);
1078 KiAcquireApcLock(IN PKTHREAD Thread
,
1079 IN PKLOCK_QUEUE_HANDLE Handle
)
1081 /* Acquire the lock and raise to synchronization level */
1082 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
1087 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
1088 IN PKLOCK_QUEUE_HANDLE Handle
)
1090 /* Acquire the lock */
1091 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
1096 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
1097 IN PKLOCK_QUEUE_HANDLE Handle
)
1099 /* Acquire the lock */
1100 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
1105 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1107 /* Release the lock */
1108 KeReleaseInStackQueuedSpinLock(Handle
);
1113 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1115 /* Release the lock */
1116 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1121 KiAcquireProcessLock(IN PKPROCESS Process
,
1122 IN PKLOCK_QUEUE_HANDLE Handle
)
1124 /* Acquire the lock and raise to synchronization level */
1125 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
1130 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1132 /* Release the lock */
1133 KeReleaseInStackQueuedSpinLock(Handle
);
1138 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1140 /* Release the lock */
1141 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1146 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
1147 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1149 /* Check if we were called from a threaded DPC */
1150 if (KeGetCurrentPrcb()->DpcThreadActive
)
1152 /* Lock the Queue, we're not at DPC level */
1153 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
1157 /* We must be at DPC level, acquire the lock safely */
1158 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1159 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
1166 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1168 /* Check if we were called from a threaded DPC */
1169 if (KeGetCurrentPrcb()->DpcThreadActive
)
1171 /* Unlock the Queue, we're not at DPC level */
1172 KeReleaseInStackQueuedSpinLock(DeviceLock
);
1176 /* We must be at DPC level, release the lock safely */
1177 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1178 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
1183 // This routine queues a thread that is ready on the PRCB's ready lists.
1184 // If this thread cannot currently run on this CPU, then the thread is
1185 // added to the deferred ready list instead.
1187 // This routine must be entered with the PRCB lock held and it will exit
1188 // with the PRCB lock released!
1192 KxQueueReadyThread(IN PKTHREAD Thread
,
1199 ASSERT(Prcb
== KeGetCurrentPrcb());
1200 ASSERT(Thread
->State
== Running
);
1201 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1203 /* Check if this thread is allowed to run in this CPU */
1205 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1210 /* Set thread ready for execution */
1211 Thread
->State
= Ready
;
1213 /* Save current priority and if someone had pre-empted it */
1214 Priority
= Thread
->Priority
;
1215 Preempted
= Thread
->Preempted
;
1217 /* We're not pre-empting now, and set the wait time */
1218 Thread
->Preempted
= FALSE
;
1219 Thread
->WaitTime
= KeTickCount
.LowPart
;
1222 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1224 /* Insert this thread in the appropriate order */
1225 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1226 &Thread
->WaitListEntry
) :
1227 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1228 &Thread
->WaitListEntry
);
1230 /* Update the ready summary */
1231 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1234 ASSERT(Priority
== Thread
->Priority
);
1236 /* Release the PRCB lock */
1237 KiReleasePrcbLock(Prcb
);
1241 /* Otherwise, prepare this thread to be deferred */
1242 Thread
->State
= DeferredReady
;
1243 Thread
->DeferredProcessor
= Prcb
->Number
;
1245 /* Release the lock and defer scheduling */
1246 KiReleasePrcbLock(Prcb
);
1247 KiDeferredReadyThread(Thread
);
1252 // This routine scans for an appropriate ready thread to select at the
1253 // given priority and for the given CPU.
1257 KiSelectReadyThread(IN KPRIORITY Priority
,
1260 LONG PriorityMask
, PrioritySet
, HighPriority
;
1261 PLIST_ENTRY ListEntry
;
1264 /* Save the current mask and get the priority set for the CPU */
1265 PriorityMask
= Priority
;
1266 PrioritySet
= Prcb
->ReadySummary
>> (UCHAR
)Priority
;
1267 if (!PrioritySet
) return NULL
;
1269 /* Get the highest priority possible */
1270 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1271 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1272 HighPriority
+= PriorityMask
;
1274 /* Make sure the list isn't at highest priority */
1275 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1277 /* Get the first thread on the list */
1278 ListEntry
= &Prcb
->DispatcherReadyListHead
[HighPriority
];
1279 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1281 /* Make sure this thread is here for a reason */
1282 ASSERT(HighPriority
== Thread
->Priority
);
1283 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1284 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1286 /* Remove it from the list */
1287 RemoveEntryList(&Thread
->WaitListEntry
);
1288 if (IsListEmpty(&Thread
->WaitListEntry
))
1290 /* The list is empty now, reset the ready summary */
1291 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1294 /* Sanity check and return the thread */
1295 ASSERT((Thread
== NULL
) ||
1296 (Thread
->BasePriority
== 0) ||
1297 (Thread
->Priority
!= 0));
1302 // This routine computes the new priority for a thread. It is only valid for
1303 // threads with priorities in the dynamic priority range.
1307 KiComputeNewPriority(IN PKTHREAD Thread
)
1311 /* Priority sanity checks */
1312 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1313 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1314 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1315 TRUE
: (Thread
->PriorityDecrement
== 0));
1317 /* Get the current priority */
1318 Priority
= Thread
->Priority
;
1319 if (Priority
< LOW_REALTIME_PRIORITY
)
1321 /* Decrease priority by the priority decrement */
1322 Priority
-= (Thread
->PriorityDecrement
+ 1);
1324 /* Don't go out of bounds */
1325 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1327 /* Reset the priority decrement */
1328 Thread
->PriorityDecrement
= 0;
1332 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1334 /* Return the new priority */
1340 KeGetCurrentThread(VOID
)
1342 /* Return the current thread */
1343 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
1348 KeGetPreviousMode(VOID
)
1350 /* Return the current mode */
1351 return KeGetCurrentThread()->PreviousMode
;