2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
17 // Sanitizes a selector
21 Ke386SanitizeSeg(IN ULONG Cs
,
22 IN KPROCESSOR_MODE Mode
)
25 // Check if we're in kernel-mode, and force CPL 0 if so.
26 // Otherwise, force CPL 3.
28 return ((Mode
== KernelMode
) ?
29 (Cs
& (0xFFFF & ~RPL_MASK
)) :
30 (RPL_MASK
| (Cs
& 0xFFFF)));
38 Ke386SanitizeFlags(IN ULONG Eflags
,
39 IN KPROCESSOR_MODE Mode
)
42 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
43 // Otherwise, also force interrupt mask on.
45 return ((Mode
== KernelMode
) ?
46 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
47 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
51 // Gets a DR register from a CONTEXT structure
55 KiDrFromContext(IN ULONG Dr
,
58 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
62 // Gets a DR register from a KTRAP_FRAME structure
66 KiDrFromTrapFrame(IN ULONG Dr
,
67 IN PKTRAP_FRAME TrapFrame
)
69 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
77 Ke386SanitizeDr(IN PVOID DrAddress
,
78 IN KPROCESSOR_MODE Mode
)
81 // Check if we're in kernel-mode, and return the address directly if so.
82 // Otherwise, make sure it's not inside the kernel-mode address space.
83 // If it is, then clear the address.
85 return ((Mode
== KernelMode
) ? DrAddress
:
86 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
90 // Enters a Guarded Region
92 #define KeEnterGuardedRegion() \
94 PKTHREAD _Thread = KeGetCurrentThread(); \
97 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
98 ASSERT(_Thread == KeGetCurrentThread()); \
99 ASSERT((_Thread->SpecialApcDisable <= 0) && \
100 (_Thread->SpecialApcDisable != -32768)); \
102 /* Disable Special APCs */ \
103 _Thread->SpecialApcDisable--; \
107 // Leaves a Guarded Region
109 #define KeLeaveGuardedRegion() \
111 PKTHREAD _Thread = KeGetCurrentThread(); \
113 /* Sanity checks */ \
114 ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
115 ASSERT(_Thread == KeGetCurrentThread()); \
116 ASSERT(_Thread->SpecialApcDisable < 0); \
118 /* Leave region and check if APCs are OK now */ \
119 if (!(++_Thread->SpecialApcDisable)) \
121 /* Check for Kernel APCs on the list */ \
122 if (!IsListEmpty(&_Thread->ApcState. \
123 ApcListHead[KernelMode])) \
125 /* Check for APC Delivery */ \
126 KiCheckForKernelApcDelivery(); \
132 // TODO: Guarded Mutex Routines
136 // Enters a Critical Region
138 #define KeEnterCriticalRegion() \
140 PKTHREAD _Thread = KeGetCurrentThread(); \
142 /* Sanity checks */ \
143 ASSERT(_Thread == KeGetCurrentThread()); \
144 ASSERT((_Thread->KernelApcDisable <= 0) && \
145 (_Thread->KernelApcDisable != -32768)); \
147 /* Disable Kernel APCs */ \
148 _Thread->KernelApcDisable--; \
152 // Leaves a Critical Region
154 #define KeLeaveCriticalRegion() \
156 PKTHREAD _Thread = KeGetCurrentThread(); \
158 /* Sanity checks */ \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->KernelApcDisable < 0); \
162 /* Enable Kernel APCs */ \
163 _Thread->KernelApcDisable++; \
165 /* Check if Kernel APCs are now enabled */ \
166 if (!(_Thread->KernelApcDisable)) \
168 /* Check if we need to request an APC Delivery */ \
169 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
170 !(_Thread->SpecialApcDisable)) \
172 /* Check for the right environment */ \
173 KiCheckForKernelApcDelivery(); \
179 // Satisfies the wait of any dispatcher object
181 #define KiSatisfyObjectWait(Object, Thread) \
183 /* Special case for Mutants */ \
184 if ((Object)->Header.Type == MutantObject) \
186 /* Decrease the Signal State */ \
187 (Object)->Header.SignalState--; \
189 /* Check if it's now non-signaled */ \
190 if (!(Object)->Header.SignalState) \
192 /* Set the Owner Thread */ \
193 (Object)->OwnerThread = Thread; \
195 /* Disable APCs if needed */ \
196 Thread->KernelApcDisable -= (Object)->ApcDisable; \
198 /* Check if it's abandoned */ \
199 if ((Object)->Abandoned) \
202 (Object)->Abandoned = FALSE; \
204 /* Return Status */ \
205 Thread->WaitStatus = STATUS_ABANDONED; \
208 /* Insert it into the Mutant List */ \
209 InsertHeadList(Thread->MutantListHead.Blink, \
210 &(Object)->MutantListEntry); \
213 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
214 EventSynchronizationObject) \
216 /* Synchronization Timers and Events just get un-signaled */ \
217 (Object)->Header.SignalState = 0; \
219 else if ((Object)->Header.Type == SemaphoreObject) \
221 /* These ones can have multiple states, so we only decrease it */ \
222 (Object)->Header.SignalState--; \
227 // Satisfies the wait of a mutant dispatcher object
229 #define KiSatisfyMutantWait(Object, Thread) \
231 /* Decrease the Signal State */ \
232 (Object)->Header.SignalState--; \
234 /* Check if it's now non-signaled */ \
235 if (!(Object)->Header.SignalState) \
237 /* Set the Owner Thread */ \
238 (Object)->OwnerThread = Thread; \
240 /* Disable APCs if needed */ \
241 Thread->KernelApcDisable -= (Object)->ApcDisable; \
243 /* Check if it's abandoned */ \
244 if ((Object)->Abandoned) \
247 (Object)->Abandoned = FALSE; \
249 /* Return Status */ \
250 Thread->WaitStatus = STATUS_ABANDONED; \
253 /* Insert it into the Mutant List */ \
254 InsertHeadList(Thread->MutantListHead.Blink, \
255 &(Object)->MutantListEntry); \
260 // Satisfies the wait of any nonmutant dispatcher object
262 #define KiSatisfyNonMutantWait(Object, Thread) \
264 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
265 EventSynchronizationObject) \
267 /* Synchronization Timers and Events just get un-signaled */ \
268 (Object)->Header.SignalState = 0; \
270 else if ((Object)->Header.Type == SemaphoreObject) \
272 /* These ones can have multiple states, so we only decrease it */ \
273 (Object)->Header.SignalState--; \
278 // Recalculates the due time
282 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
283 IN PLARGE_INTEGER DueTime
,
284 IN OUT PLARGE_INTEGER NewDueTime
)
286 /* Don't do anything for absolute waits */
287 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
289 /* Otherwise, query the interrupt time and recalculate */
290 NewDueTime
->QuadPart
= KeQueryInterruptTime();
291 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
296 // Determines wether a thread should be added to the wait list
300 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
301 IN KPROCESSOR_MODE WaitMode
)
303 /* Check the required conditions */
304 if ((WaitMode
!= KernelMode
) &&
305 (Thread
->EnableStackSwap
) &&
306 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
308 /* We are go for swap */
313 /* Don't swap the thread */
319 // Adds a thread to the wait list
321 #define KiAddThreadToWaitList(Thread, Swappable) \
323 /* Make sure it's swappable */ \
326 /* Insert it into the PRCB's List */ \
327 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
328 &Thread->WaitListEntry); \
333 // Checks if a wait in progress should be interrupted by APCs or an alertable
338 KiCheckAlertability(IN PKTHREAD Thread
,
339 IN BOOLEAN Alertable
,
340 IN KPROCESSOR_MODE WaitMode
)
342 /* Check if the wait is alertable */
345 /* It is, first check if the thread is alerted in this mode */
346 if (Thread
->Alerted
[WaitMode
])
348 /* It is, so bail out of the wait */
349 Thread
->Alerted
[WaitMode
] = FALSE
;
350 return STATUS_ALERTED
;
352 else if ((WaitMode
!= KernelMode
) &&
353 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
355 /* It's isn't, but this is a user wait with queued user APCs */
356 Thread
->ApcState
.UserApcPending
= TRUE
;
357 return STATUS_USER_APC
;
359 else if (Thread
->Alerted
[KernelMode
])
361 /* It isn't that either, but we're alered in kernel mode */
362 Thread
->Alerted
[KernelMode
] = FALSE
;
363 return STATUS_ALERTED
;
366 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
368 /* Not alertable, but this is a user wait with pending user APCs */
369 return STATUS_USER_APC
;
372 /* Otherwise, we're fine */
373 return STATUS_WAIT_0
;
378 KxSetTimerForThreadWait(IN PKTIMER Timer
,
379 IN LARGE_INTEGER Interval
)
381 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
383 /* Check the timer's interval to see if it's absolute */
384 Timer
->Header
.Absolute
= FALSE
;
385 if (!Timer
->Period
) Timer
->Header
.SignalState
= FALSE
;
386 if (Interval
.HighPart
>= 0)
388 /* Get the system time and calculate the relative time */
389 KeQuerySystemTime(&SystemTime
);
390 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
391 Timer
->Header
.Absolute
= TRUE
;
393 /* Check if we've already expired */
394 if (TimeDifference
.HighPart
>= 0)
396 /* Reset everything */
397 Timer
->DueTime
.QuadPart
= 0;
398 Timer
->Header
.SignalState
= TRUE
;
403 /* Update the interval */
404 Interval
= TimeDifference
;
408 /* Calculate the due time */
409 InterruptTime
.QuadPart
= KeQueryInterruptTime();
410 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
413 #define KxDelayThreadWait() \
415 /* Setup the Wait Block */ \
416 Thread->WaitBlockList = TimerBlock; \
418 /* Setup the timer */ \
419 KxSetTimerForThreadWait(Timer, *Interval); \
421 /* Save the due time for the caller */ \
422 DueTime.QuadPart = Timer->DueTime.QuadPart; \
424 /* Link the timer to this Wait Block */ \
425 TimerBlock->NextWaitBlock = TimerBlock; \
426 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
427 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
429 /* Clear wait status */ \
430 Thread->WaitStatus = STATUS_SUCCESS; \
432 /* Setup wait fields */ \
433 Thread->Alertable = Alertable; \
434 Thread->WaitReason = DelayExecution; \
435 Thread->WaitMode = WaitMode; \
437 /* Check if we can swap the thread's stack */ \
438 Thread->WaitListEntry.Flink = NULL; \
439 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
441 /* Set the wait time */ \
442 Thread->WaitTime = KeTickCount.LowPart;
444 #define KxMultiThreadWait() \
445 /* Link wait block array to the thread */ \
446 Thread->WaitBlockList = WaitBlockArray; \
448 /* Reset the index */ \
451 /* Loop wait blocks */ \
454 /* Fill out the wait block */ \
455 WaitBlock = &WaitBlockArray[Index]; \
456 WaitBlock->Object = Object[Index]; \
457 WaitBlock->WaitKey = Index; \
458 WaitBlock->WaitType = WaitType; \
459 WaitBlock->Thread = Thread; \
461 /* Link to next block */ \
462 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
464 } while (Index < Count); \
466 /* Link the last block */ \
467 WaitBlock->NextWaitBlock = WaitBlockArray; \
469 /* Set default wait status */ \
470 Thread->WaitStatus = STATUS_WAIT_0; \
472 /* Check if we have a timer */ \
475 /* Link to the block */ \
476 TimerBlock->NextWaitBlock = WaitBlockArray; \
478 /* Setup the timer */ \
479 KxSetTimerForThreadWait(Timer, *Timeout); \
481 /* Save the due time for the caller */ \
482 DueTime.QuadPart = Timer->DueTime.QuadPart; \
484 /* Initialize the list */ \
485 InitializeListHead(&Timer->Header.WaitListHead); \
488 /* Set wait settings */ \
489 Thread->Alertable = Alertable; \
490 Thread->WaitMode = WaitMode; \
491 Thread->WaitReason = WaitReason; \
493 /* Check if we can swap the thread's stack */ \
494 Thread->WaitListEntry.Flink = NULL; \
495 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
497 /* Set the wait time */ \
498 Thread->WaitTime = KeTickCount.LowPart;
500 #define KxSingleThreadWait() \
501 /* Setup the Wait Block */ \
502 Thread->WaitBlockList = WaitBlock; \
503 WaitBlock->WaitKey = STATUS_SUCCESS; \
504 WaitBlock->Object = Object; \
505 WaitBlock->WaitType = WaitAny; \
507 /* Clear wait status */ \
508 Thread->WaitStatus = STATUS_SUCCESS; \
510 /* Check if we have a timer */ \
513 /* Setup the timer */ \
514 KxSetTimerForThreadWait(Timer, *Timeout); \
516 /* Save the due time for the caller */ \
517 DueTime.QuadPart = Timer->DueTime.QuadPart; \
519 /* Pointer to timer block */ \
520 WaitBlock->NextWaitBlock = TimerBlock; \
521 TimerBlock->NextWaitBlock = WaitBlock; \
523 /* Link the timer to this Wait Block */ \
524 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
525 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
529 /* No timer block, just ourselves */ \
530 WaitBlock->NextWaitBlock = WaitBlock; \
533 /* Set wait settings */ \
534 Thread->Alertable = Alertable; \
535 Thread->WaitMode = WaitMode; \
536 Thread->WaitReason = WaitReason; \
538 /* Check if we can swap the thread's stack */ \
539 Thread->WaitListEntry.Flink = NULL; \
540 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
542 /* Set the wait time */ \
543 Thread->WaitTime = KeTickCount.LowPart;
545 #define KxQueueThreadWait() \
546 /* Setup the Wait Block */ \
547 Thread->WaitBlockList = WaitBlock; \
548 WaitBlock->WaitKey = STATUS_SUCCESS; \
549 WaitBlock->Object = Queue; \
550 WaitBlock->WaitType = WaitAny; \
551 WaitBlock->Thread = Thread; \
553 /* Clear wait status */ \
554 Thread->WaitStatus = STATUS_SUCCESS; \
556 /* Check if we have a timer */ \
559 /* Setup the timer */ \
560 KxSetTimerForThreadWait(Timer, *Timeout); \
562 /* Save the due time for the caller */ \
563 DueTime.QuadPart = Timer->DueTime.QuadPart; \
565 /* Pointer to timer block */ \
566 WaitBlock->NextWaitBlock = TimerBlock; \
567 TimerBlock->NextWaitBlock = WaitBlock; \
569 /* Link the timer to this Wait Block */ \
570 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
571 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
575 /* No timer block, just ourselves */ \
576 WaitBlock->NextWaitBlock = WaitBlock; \
579 /* Set wait settings */ \
580 Thread->Alertable = FALSE; \
581 Thread->WaitMode = WaitMode; \
582 Thread->WaitReason = WrQueue; \
584 /* Check if we can swap the thread's stack */ \
585 Thread->WaitListEntry.Flink = NULL; \
586 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
588 /* Set the wait time */ \
589 Thread->WaitTime = KeTickCount.LowPart;
596 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
597 IN KPRIORITY Increment
)
599 PLIST_ENTRY WaitEntry
, WaitList
;
600 PKWAIT_BLOCK WaitBlock
;
604 /* Loop the Wait Entries */
605 WaitList
= &Object
->WaitListHead
;
606 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
607 WaitEntry
= WaitList
->Flink
;
610 /* Get the current wait block */
611 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
613 /* Get the waiting thread */
614 WaitThread
= WaitBlock
->Thread
;
616 /* Check the current Wait Mode */
617 if (WaitBlock
->WaitType
== WaitAny
)
619 /* Use the actual wait key */
620 WaitKey
= WaitBlock
->WaitKey
;
624 /* Otherwise, use STATUS_KERNEL_APC */
625 WaitKey
= STATUS_KERNEL_APC
;
628 /* Unwait the thread */
629 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
632 WaitEntry
= WaitList
->Flink
;
633 } while (WaitEntry
!= WaitList
);
637 // Unwaits a Thread waiting on an event
641 KxUnwaitThreadForEvent(IN PKEVENT Event
,
642 IN KPRIORITY Increment
)
644 PLIST_ENTRY WaitEntry
, WaitList
;
645 PKWAIT_BLOCK WaitBlock
;
648 /* Loop the Wait Entries */
649 WaitList
= &Event
->Header
.WaitListHead
;
650 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
651 WaitEntry
= WaitList
->Flink
;
654 /* Get the current wait block */
655 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
657 /* Get the waiting thread */
658 WaitThread
= WaitBlock
->Thread
;
660 /* Check the current Wait Mode */
661 if (WaitBlock
->WaitType
== WaitAny
)
664 Event
->Header
.SignalState
= 0;
666 /* Un-signal the event and unwait the thread */
667 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
671 /* Unwait the thread with STATUS_KERNEL_APC */
672 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
675 WaitEntry
= WaitList
->Flink
;
676 } while (WaitEntry
!= WaitList
);
681 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
685 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
687 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
688 UNREFERENCED_PARAMETER(SpinLock
);
692 // Spinlock Release at IRQL >= DISPATCH_LEVEL
696 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
698 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
699 UNREFERENCED_PARAMETER(SpinLock
);
703 // This routine protects against multiple CPU acquires, it's meaningless on UP.
707 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
709 UNREFERENCED_PARAMETER(Object
);
713 // This routine protects against multiple CPU acquires, it's meaningless on UP.
717 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
719 UNREFERENCED_PARAMETER(Object
);
724 KiAcquireDispatcherLock(VOID
)
726 /* Raise to DPC level */
727 return KeRaiseIrqlToDpcLevel();
732 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
734 /* Just exit the dispatcher */
735 KiExitDispatcher(OldIrql
);
740 KiAcquireDispatcherLockAtDpcLevel(VOID
)
742 /* This is a no-op at DPC Level for UP systems */
748 KiReleaseDispatcherLockFromDpcLevel(VOID
)
750 /* This is a no-op at DPC Level for UP systems */
755 // This routine makes the thread deferred ready on the boot CPU.
759 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
761 /* Set the thread to deferred state and boot CPU */
762 Thread
->State
= DeferredReady
;
763 Thread
->DeferredProcessor
= 0;
765 /* Make the thread ready immediately */
766 KiDeferredReadyThread(Thread
);
771 KiRescheduleThread(IN BOOLEAN NewThread
,
774 /* This is meaningless on UP systems */
775 UNREFERENCED_PARAMETER(NewThread
);
776 UNREFERENCED_PARAMETER(Cpu
);
780 // This routine protects against multiple CPU acquires, it's meaningless on UP.
784 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
786 UNREFERENCED_PARAMETER(Thread
);
790 // This routine protects against multiple CPU acquires, it's meaningless on UP.
794 KiAcquirePrcbLock(IN PKPRCB Prcb
)
796 UNREFERENCED_PARAMETER(Prcb
);
800 // This routine protects against multiple CPU acquires, it's meaningless on UP.
804 KiReleasePrcbLock(IN PKPRCB Prcb
)
806 UNREFERENCED_PARAMETER(Prcb
);
810 // This routine protects against multiple CPU acquires, it's meaningless on UP.
814 KiAcquireThreadLock(IN PKTHREAD Thread
)
816 UNREFERENCED_PARAMETER(Thread
);
820 // This routine protects against multiple CPU acquires, it's meaningless on UP.
824 KiReleaseThreadLock(IN PKTHREAD Thread
)
826 UNREFERENCED_PARAMETER(Thread
);
830 // This routine protects against multiple CPU acquires, it's meaningless on UP.
834 KiTryThreadLock(IN PKTHREAD Thread
)
836 UNREFERENCED_PARAMETER(Thread
);
842 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
844 /* There are no deferred ready lists on UP systems */
845 UNREFERENCED_PARAMETER(Prcb
);
850 KiRundownThread(IN PKTHREAD Thread
)
852 /* Check if this is the NPX Thread */
853 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
856 KeGetCurrentPrcb()->NpxThread
= NULL
;
863 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
866 /* We deliver instantly on UP */
867 UNREFERENCED_PARAMETER(NeedApc
);
868 UNREFERENCED_PARAMETER(Processor
);
873 KiAcquireTimerLock(IN ULONG Hand
)
875 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
877 /* Nothing to do on UP */
878 UNREFERENCED_PARAMETER(Hand
);
884 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
886 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
888 /* Nothing to do on UP */
889 UNREFERENCED_PARAMETER(LockQueue
);
895 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
899 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
903 /* Try to acquire it */
904 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
906 /* Value changed... wait until it's locked */
907 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
910 /* On debug builds, we use a much slower but useful routine */
911 Kii386SpinOnSpinLock(SpinLock
, 5);
913 /* Otherwise, just yield and keep looping */
921 /* On debug builds, we OR in the KTHREAD */
922 *SpinLock
= KeGetCurrentThread() | 1;
924 /* All is well, break out */
931 // Spinlock Release at IRQL >= DISPATCH_LEVEL
935 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
938 /* Make sure that the threads match */
939 if ((KeGetCurrentThread() | 1) != *SpinLock
)
941 /* They don't, bugcheck */
942 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, SpinLock
, 0, 0, 0);
946 InterlockedAnd(SpinLock
, 0);
951 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
953 LONG OldValue
, NewValue
;
955 /* Make sure we're at a safe level to touch the lock */
956 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
958 /* Start acquire loop */
961 /* Loop until the other CPU releases it */
962 while ((UCHAR
)Object
->Lock
& KOBJECT_LOCK_BIT
)
964 /* Let the CPU know that this is a loop */
968 /* Try acquiring the lock now */
969 NewValue
= InterlockedCompareExchange(&Object
->Lock
,
970 OldValue
| KOBJECT_LOCK_BIT
,
972 } while (NewValue
!= OldValue
);
977 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
979 /* Make sure we're at a safe level to touch the lock */
980 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
983 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
988 KiAcquireDispatcherLock(VOID
)
990 /* Raise to synchronization level and acquire the dispatcher lock */
991 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
996 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
998 /* First release the lock */
999 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
1000 LockQueue
[LockQueueDispatcherLock
]);
1002 /* Then exit the dispatcher */
1003 KiExitDispatcher(OldIrql
);
1007 // This routine inserts a thread into the deferred ready list of the given CPU
1011 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
1013 PKPRCB Prcb
= KeGetCurrentPrcb();
1015 /* Set the thread to deferred state and CPU */
1016 Thread
->State
= DeferredReady
;
1017 Thread
->DeferredProcessor
= Prcb
->Number
;
1019 /* Add it on the list */
1020 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
1025 KiRescheduleThread(IN BOOLEAN NewThread
,
1028 /* Check if a new thread needs to be scheduled on a different CPU */
1029 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
1031 /* Send an IPI to request delivery */
1032 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
1037 // This routine sets the current thread in a swap busy state, which ensure that
1038 // nobody else tries to swap it concurrently.
1042 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
1044 /* Make sure nobody already set it */
1045 ASSERT(Thread
->SwapBusy
== FALSE
);
1047 /* Set it ourselves */
1048 Thread
->SwapBusy
= TRUE
;
1052 // This routine acquires the PRCB lock so that only one caller can touch
1053 // volatile PRCB data.
1055 // Since this is a simple optimized spin-lock, it must be be only acquired
1056 // at dispatcher level or higher!
1060 KiAcquirePrcbLock(IN PKPRCB Prcb
)
1062 /* Make sure we're at a safe level to touch the PRCB lock */
1063 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
1065 /* Start acquire loop */
1068 /* Acquire the lock and break out if we acquired it first */
1069 if (!InterlockedExchange(&Prcb
->PrcbLock
, 1)) break;
1071 /* Loop until the other CPU releases it */
1074 /* Let the CPU know that this is a loop */
1076 } while (Prcb
->PrcbLock
);
1081 // This routine releases the PRCB lock so that other callers can touch
1082 // volatile PRCB data.
1084 // Since this is a simple optimized spin-lock, it must be be only acquired
1085 // at dispatcher level or higher!
1089 KiReleasePrcbLock(IN PKPRCB Prcb
)
1091 /* Make sure it's acquired! */
1092 ASSERT(Prcb
->PrcbLock
!= 0);
1095 InterlockedAnd(&Prcb
->PrcbLock
, 0);
1099 // This routine acquires the thread lock so that only one caller can touch
1100 // volatile thread data.
1102 // Since this is a simple optimized spin-lock, it must be be only acquired
1103 // at dispatcher level or higher!
1107 KiAcquireThreadLock(IN PKTHREAD Thread
)
1109 /* Make sure we're at a safe level to touch the thread lock */
1110 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
1112 /* Start acquire loop */
1115 /* Acquire the lock and break out if we acquired it first */
1116 if (!InterlockedExchange(&Thread
->ThreadLock
, 1)) break;
1118 /* Loop until the other CPU releases it */
1121 /* Let the CPU know that this is a loop */
1123 } while (Thread
->ThreadLock
);
1128 // This routine releases the thread lock so that other callers can touch
1129 // volatile thread data.
1131 // Since this is a simple optimized spin-lock, it must be be only acquired
1132 // at dispatcher level or higher!
1136 KiReleaseThreadLock(IN PKTHREAD Thread
)
1139 InterlockedAnd(&Thread
->ThreadLock
, 0);
1144 KiTryThreadLock(IN PKTHREAD Thread
)
1148 /* If the lock isn't acquired, return false */
1149 if (!Thread
->ThreadLock
) return FALSE
;
1151 /* Otherwise, try to acquire it and check the result */
1153 Value
= InterlockedExchange(&Thread
->ThreadLock
, &Value
);
1155 /* Return the lock state */
1156 return (Value
== TRUE
);
1161 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
1163 /* Scan the deferred ready lists if required */
1164 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
1169 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
1172 /* Check if we need to request APC delivery */
1175 /* Check if it's on another CPU */
1176 if (KeGetPcr()->Number
!= Cpu
)
1178 /* Send an IPI to request delivery */
1179 KiIpiSendRequest(AFFINITY_MASK(Cpu
), IPI_DPC
);
1183 /* Request a software interrupt */
1184 HalRequestSoftwareInterrupt(APC_LEVEL
);
1193 KiAcquireApcLock(IN PKTHREAD Thread
,
1194 IN PKLOCK_QUEUE_HANDLE Handle
)
1196 /* Acquire the lock and raise to synchronization level */
1197 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
1202 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
1203 IN PKLOCK_QUEUE_HANDLE Handle
)
1205 /* Acquire the lock */
1206 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
1211 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
1212 IN PKLOCK_QUEUE_HANDLE Handle
)
1214 /* Acquire the lock */
1215 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
1220 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1222 /* Release the lock */
1223 KeReleaseInStackQueuedSpinLock(Handle
);
1228 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1230 /* Release the lock */
1231 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1236 KiAcquireProcessLock(IN PKPROCESS Process
,
1237 IN PKLOCK_QUEUE_HANDLE Handle
)
1239 /* Acquire the lock and raise to synchronization level */
1240 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
1245 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
1247 /* Release the lock */
1248 KeReleaseInStackQueuedSpinLock(Handle
);
1253 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
1255 /* Release the lock */
1256 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
1261 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
1262 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1264 /* Check if we were called from a threaded DPC */
1265 if (KeGetCurrentPrcb()->DpcThreadActive
)
1267 /* Lock the Queue, we're not at DPC level */
1268 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
1272 /* We must be at DPC level, acquire the lock safely */
1273 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1274 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
1281 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
1283 /* Check if we were called from a threaded DPC */
1284 if (KeGetCurrentPrcb()->DpcThreadActive
)
1286 /* Unlock the Queue, we're not at DPC level */
1287 KeReleaseInStackQueuedSpinLock(DeviceLock
);
1291 /* We must be at DPC level, release the lock safely */
1292 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1293 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
1298 // This routine queues a thread that is ready on the PRCB's ready lists.
1299 // If this thread cannot currently run on this CPU, then the thread is
1300 // added to the deferred ready list instead.
1302 // This routine must be entered with the PRCB lock held and it will exit
1303 // with the PRCB lock released!
1307 KxQueueReadyThread(IN PKTHREAD Thread
,
1314 ASSERT(Prcb
== KeGetCurrentPrcb());
1315 ASSERT(Thread
->State
== Running
);
1316 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1318 /* Check if this thread is allowed to run in this CPU */
1320 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1325 /* Set thread ready for execution */
1326 Thread
->State
= Ready
;
1328 /* Save current priority and if someone had pre-empted it */
1329 Priority
= Thread
->Priority
;
1330 Preempted
= Thread
->Preempted
;
1332 /* We're not pre-empting now, and set the wait time */
1333 Thread
->Preempted
= FALSE
;
1334 Thread
->WaitTime
= KeTickCount
.LowPart
;
1337 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1339 /* Insert this thread in the appropriate order */
1340 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1341 &Thread
->WaitListEntry
) :
1342 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1343 &Thread
->WaitListEntry
);
1345 /* Update the ready summary */
1346 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1349 ASSERT(Priority
== Thread
->Priority
);
1351 /* Release the PRCB lock */
1352 KiReleasePrcbLock(Prcb
);
1356 /* Otherwise, prepare this thread to be deferred */
1357 Thread
->State
= DeferredReady
;
1358 Thread
->DeferredProcessor
= Prcb
->Number
;
1360 /* Release the lock and defer scheduling */
1361 KiReleasePrcbLock(Prcb
);
1362 KiDeferredReadyThread(Thread
);
1367 // This routine scans for an appropriate ready thread to select at the
1368 // given priority and for the given CPU.
1372 KiSelectReadyThread(IN KPRIORITY Priority
,
1375 ULONG PrioritySet
, HighPriority
;
1376 PLIST_ENTRY ListEntry
;
1377 PKTHREAD Thread
= NULL
;
1379 /* Save the current mask and get the priority set for the CPU */
1380 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1381 if (!PrioritySet
) goto Quickie
;
1383 /* Get the highest priority possible */
1384 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1385 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1386 HighPriority
+= Priority
;
1388 /* Make sure the list isn't empty at the highest priority */
1389 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1391 /* Get the first thread on the list */
1392 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1393 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1395 /* Make sure this thread is here for a reason */
1396 ASSERT(HighPriority
== Thread
->Priority
);
1397 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1398 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1400 /* Remove it from the list */
1401 if (RemoveEntryList(&Thread
->WaitListEntry
))
1403 /* The list is empty now, reset the ready summary */
1404 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1407 /* Sanity check and return the thread */
1409 ASSERT((Thread
== NULL
) ||
1410 (Thread
->BasePriority
== 0) ||
1411 (Thread
->Priority
!= 0));
1416 // This routine computes the new priority for a thread. It is only valid for
1417 // threads with priorities in the dynamic priority range.
1421 KiComputeNewPriority(IN PKTHREAD Thread
,
1422 IN SCHAR Adjustment
)
1426 /* Priority sanity checks */
1427 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1428 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1429 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1430 TRUE
: (Thread
->PriorityDecrement
== 0));
1432 /* Get the current priority */
1433 Priority
= Thread
->Priority
;
1434 if (Priority
< LOW_REALTIME_PRIORITY
)
1436 /* Decrease priority by the priority decrement */
1437 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1439 /* Don't go out of bounds */
1440 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1442 /* Reset the priority decrement */
1443 Thread
->PriorityDecrement
= 0;
1447 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1449 /* Return the new priority */
1455 KeGetCurrentThread(VOID
)
1457 /* Return the current thread */
1458 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
1463 KeGetPreviousMode(VOID
)
1465 /* Return the current mode */
1466 return KeGetCurrentThread()->PreviousMode
;
1471 KiInsertWaitTimer(IN PKTIMER Timer
)
1473 /* Set default data */
1474 Timer
->Header
.Inserted
= TRUE
;
1475 if (!Timer
->Period
) Timer
->Header
.SignalState
= FALSE
;
1477 /* Now insert it into the Timer List */
1478 InsertAscendingList(&KiTimerListHead
,