2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/include/ke_x.h
5 * PURPOSE: Internal Inlined Functions for the Kernel
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 // Thread Dispatcher Header DebugActive Mask
12 #define DR_MASK(x) 1 << x
13 #define DR_ACTIVE_MASK 0x10
14 #define DR_REG_MASK 0x4F
18 // Sanitizes a selector
22 Ke386SanitizeSeg(IN ULONG Cs
,
23 IN KPROCESSOR_MODE Mode
)
26 // Check if we're in kernel-mode, and force CPL 0 if so.
27 // Otherwise, force CPL 3.
29 return ((Mode
== KernelMode
) ?
30 (Cs
& (0xFFFF & ~RPL_MASK
)) :
31 (RPL_MASK
| (Cs
& 0xFFFF)));
39 Ke386SanitizeFlags(IN ULONG Eflags
,
40 IN KPROCESSOR_MODE Mode
)
43 // Check if we're in kernel-mode, and sanitize EFLAGS if so.
44 // Otherwise, also force interrupt mask on.
46 return ((Mode
== KernelMode
) ?
47 (Eflags
& (EFLAGS_USER_SANITIZE
| EFLAGS_INTERRUPT_MASK
)) :
48 (EFLAGS_INTERRUPT_MASK
| (Eflags
& EFLAGS_USER_SANITIZE
)));
52 // Gets a DR register from a CONTEXT structure
56 KiDrFromContext(IN ULONG Dr
,
59 return *(PVOID
*)((ULONG_PTR
)Context
+ KiDebugRegisterContextOffsets
[Dr
]);
63 // Gets a DR register from a KTRAP_FRAME structure
67 KiDrFromTrapFrame(IN ULONG Dr
,
68 IN PKTRAP_FRAME TrapFrame
)
70 return (PVOID
*)((ULONG_PTR
)TrapFrame
+ KiDebugRegisterTrapOffsets
[Dr
]);
78 Ke386SanitizeDr(IN PVOID DrAddress
,
79 IN KPROCESSOR_MODE Mode
)
82 // Check if we're in kernel-mode, and return the address directly if so.
83 // Otherwise, make sure it's not inside the kernel-mode address space.
84 // If it is, then clear the address.
86 return ((Mode
== KernelMode
) ? DrAddress
:
87 (DrAddress
<= MM_HIGHEST_USER_ADDRESS
) ? DrAddress
: 0);
94 KeGetCurrentThread(VOID
)
97 /* Return the current thread */
98 return ((PKIPCR
)KeGetPcr())->PrcbData
.CurrentThread
;
99 #elif defined (_M_AMD64)
100 return (PRKTHREAD
)__readgsqword(FIELD_OFFSET(KIPCR
, Prcb
.CurrentThread
));
102 PKPRCB Prcb
= KeGetCurrentPrcb();
103 return Prcb
->CurrentThread
;
109 KeGetPreviousMode(VOID
)
111 /* Return the current mode */
112 return KeGetCurrentThread()->PreviousMode
;
118 KeFlushProcessTb(VOID
)
120 /* Flush the TLB by resetting CR3 */
122 __asm__("sync\n\tisync\n\t");
125 // We need to implement this!
127 ASSERTMSG("Need ARM flush routine\n", FALSE
);
129 __writecr3(__readcr3());
134 // Enters a Guarded Region
136 #define KeEnterGuardedRegion() \
138 PKTHREAD _Thread = KeGetCurrentThread(); \
140 /* Sanity checks */ \
141 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
142 ASSERT(_Thread == KeGetCurrentThread()); \
143 ASSERT((_Thread->SpecialApcDisable <= 0) && \
144 (_Thread->SpecialApcDisable != -32768)); \
146 /* Disable Special APCs */ \
147 _Thread->SpecialApcDisable--; \
151 // Leaves a Guarded Region
153 #define KeLeaveGuardedRegion() \
155 PKTHREAD _Thread = KeGetCurrentThread(); \
157 /* Sanity checks */ \
158 ASSERT(KeGetCurrentIrql() <= APC_LEVEL); \
159 ASSERT(_Thread == KeGetCurrentThread()); \
160 ASSERT(_Thread->SpecialApcDisable < 0); \
162 /* Leave region and check if APCs are OK now */ \
163 if (!(++_Thread->SpecialApcDisable)) \
165 /* Check for Kernel APCs on the list */ \
166 if (!IsListEmpty(&_Thread->ApcState. \
167 ApcListHead[KernelMode])) \
169 /* Check for APC Delivery */ \
170 KiCheckForKernelApcDelivery(); \
176 // TODO: Guarded Mutex Routines
180 // Enters a Critical Region
182 #define KeEnterCriticalRegion() \
184 PKTHREAD _Thread = KeGetCurrentThread(); \
186 /* Sanity checks */ \
187 ASSERT(_Thread == KeGetCurrentThread()); \
188 ASSERT((_Thread->KernelApcDisable <= 0) && \
189 (_Thread->KernelApcDisable != -32768)); \
191 /* Disable Kernel APCs */ \
192 _Thread->KernelApcDisable--; \
196 // Leaves a Critical Region
198 #define KeLeaveCriticalRegion() \
200 PKTHREAD _Thread = KeGetCurrentThread(); \
202 /* Sanity checks */ \
203 ASSERT(_Thread == KeGetCurrentThread()); \
204 ASSERT(_Thread->KernelApcDisable < 0); \
206 /* Enable Kernel APCs */ \
207 _Thread->KernelApcDisable++; \
209 /* Check if Kernel APCs are now enabled */ \
210 if (!(_Thread->KernelApcDisable)) \
212 /* Check if we need to request an APC Delivery */ \
213 if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
214 !(_Thread->SpecialApcDisable)) \
216 /* Check for the right environment */ \
217 KiCheckForKernelApcDelivery(); \
224 // Spinlock Acquire at IRQL >= DISPATCH_LEVEL
228 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
230 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
231 UNREFERENCED_PARAMETER(SpinLock
);
235 // Spinlock Release at IRQL >= DISPATCH_LEVEL
239 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
241 /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
242 UNREFERENCED_PARAMETER(SpinLock
);
246 // This routine protects against multiple CPU acquires, it's meaningless on UP.
250 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
252 UNREFERENCED_PARAMETER(Object
);
256 // This routine protects against multiple CPU acquires, it's meaningless on UP.
260 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
262 UNREFERENCED_PARAMETER(Object
);
267 KiAcquireDispatcherLock(VOID
)
269 /* Raise to DPC level */
270 return KeRaiseIrqlToDpcLevel();
275 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
277 /* Just exit the dispatcher */
278 KiExitDispatcher(OldIrql
);
283 KiAcquireDispatcherLockAtDpcLevel(VOID
)
285 /* This is a no-op at DPC Level for UP systems */
291 KiReleaseDispatcherLockFromDpcLevel(VOID
)
293 /* This is a no-op at DPC Level for UP systems */
298 // This routine makes the thread deferred ready on the boot CPU.
302 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
304 /* Set the thread to deferred state and boot CPU */
305 Thread
->State
= DeferredReady
;
306 Thread
->DeferredProcessor
= 0;
308 /* Make the thread ready immediately */
309 KiDeferredReadyThread(Thread
);
314 KiRescheduleThread(IN BOOLEAN NewThread
,
317 /* This is meaningless on UP systems */
318 UNREFERENCED_PARAMETER(NewThread
);
319 UNREFERENCED_PARAMETER(Cpu
);
323 // This routine protects against multiple CPU acquires, it's meaningless on UP.
327 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
329 UNREFERENCED_PARAMETER(Thread
);
333 // This routine protects against multiple CPU acquires, it's meaningless on UP.
337 KiAcquirePrcbLock(IN PKPRCB Prcb
)
339 UNREFERENCED_PARAMETER(Prcb
);
343 // This routine protects against multiple CPU acquires, it's meaningless on UP.
347 KiReleasePrcbLock(IN PKPRCB Prcb
)
349 UNREFERENCED_PARAMETER(Prcb
);
353 // This routine protects against multiple CPU acquires, it's meaningless on UP.
357 KiAcquireThreadLock(IN PKTHREAD Thread
)
359 UNREFERENCED_PARAMETER(Thread
);
363 // This routine protects against multiple CPU acquires, it's meaningless on UP.
367 KiReleaseThreadLock(IN PKTHREAD Thread
)
369 UNREFERENCED_PARAMETER(Thread
);
373 // This routine protects against multiple CPU acquires, it's meaningless on UP.
377 KiTryThreadLock(IN PKTHREAD Thread
)
379 UNREFERENCED_PARAMETER(Thread
);
385 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
387 /* There are no deferred ready lists on UP systems */
388 UNREFERENCED_PARAMETER(Prcb
);
393 KiRundownThread(IN PKTHREAD Thread
)
396 /* Check if this is the NPX Thread */
397 if (KeGetCurrentPrcb()->NpxThread
== Thread
)
400 KeGetCurrentPrcb()->NpxThread
= NULL
;
408 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
411 /* We deliver instantly on UP */
412 UNREFERENCED_PARAMETER(NeedApc
);
413 UNREFERENCED_PARAMETER(Processor
);
418 KiAcquireTimerLock(IN ULONG Hand
)
420 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
422 /* Nothing to do on UP */
423 UNREFERENCED_PARAMETER(Hand
);
429 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
431 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
433 /* Nothing to do on UP */
434 UNREFERENCED_PARAMETER(LockQueue
);
440 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
444 KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock
)
448 /* Try to acquire it */
449 if (InterlockedBitTestAndSet((PLONG
)SpinLock
, 0))
451 /* Value changed... wait until it's locked */
452 while (*(volatile KSPIN_LOCK
*)SpinLock
== 1)
455 /* On debug builds, we use a much slower but useful routine */
456 //Kii386SpinOnSpinLock(SpinLock, 5);
458 /* FIXME: Do normal yield for now */
461 /* Otherwise, just yield and keep looping */
469 /* On debug builds, we OR in the KTHREAD */
470 *SpinLock
= (KSPIN_LOCK
)KeGetCurrentThread() | 1;
472 /* All is well, break out */
479 // Spinlock Release at IRQL >= DISPATCH_LEVEL
483 KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock
)
486 /* Make sure that the threads match */
487 if (((KSPIN_LOCK
)KeGetCurrentThread() | 1) != *SpinLock
)
489 /* They don't, bugcheck */
490 KeBugCheckEx(SPIN_LOCK_NOT_OWNED
, (ULONG_PTR
)SpinLock
, 0, 0, 0);
494 InterlockedAnd((PLONG
)SpinLock
, 0);
499 KiAcquireDispatcherObject(IN DISPATCHER_HEADER
* Object
)
503 /* Make sure we're at a safe level to touch the lock */
504 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
506 /* Start acquire loop */
509 /* Loop until the other CPU releases it */
512 /* Check if it got released */
513 OldValue
= Object
->Lock
;
514 if ((OldValue
& KOBJECT_LOCK_BIT
) == 0) break;
516 /* Let the CPU know that this is a loop */
520 /* Try acquiring the lock now */
521 } while (InterlockedCompareExchange(&Object
->Lock
,
522 OldValue
| KOBJECT_LOCK_BIT
,
523 OldValue
) != OldValue
);
528 KiReleaseDispatcherObject(IN DISPATCHER_HEADER
* Object
)
530 /* Make sure we're at a safe level to touch the lock */
531 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
534 InterlockedAnd(&Object
->Lock
, ~KOBJECT_LOCK_BIT
);
539 KiAcquireDispatcherLock(VOID
)
541 /* Raise to synchronization level and acquire the dispatcher lock */
542 return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock
);
547 KiReleaseDispatcherLock(IN KIRQL OldIrql
)
549 /* First release the lock */
550 KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
551 LockQueue
[LockQueueDispatcherLock
]);
553 /* Then exit the dispatcher */
554 KiExitDispatcher(OldIrql
);
559 KiAcquireDispatcherLockAtDpcLevel(VOID
)
561 /* Acquire the dispatcher lock */
562 KeAcquireQueuedSpinLockAtDpcLevel(LockQueueDispatcherLock
);
567 KiReleaseDispatcherLockFromDpcLevel(VOID
)
569 /* Release the dispatcher lock */
570 KeReleaseQueuedSpinLockFromDpcLevel(LockQueueDispatcherLock
);
574 // This routine inserts a thread into the deferred ready list of the given CPU
578 KiInsertDeferredReadyList(IN PKTHREAD Thread
)
580 PKPRCB Prcb
= KeGetCurrentPrcb();
582 /* Set the thread to deferred state and CPU */
583 Thread
->State
= DeferredReady
;
584 Thread
->DeferredProcessor
= Prcb
->Number
;
586 /* Add it on the list */
587 PushEntryList(&Prcb
->DeferredReadyListHead
, &Thread
->SwapListEntry
);
592 KiRescheduleThread(IN BOOLEAN NewThread
,
595 /* Check if a new thread needs to be scheduled on a different CPU */
596 if ((NewThread
) && !(KeGetPcr()->Number
== Cpu
))
598 /* Send an IPI to request delivery */
599 KiIpiSend(AFFINITY_MASK(Cpu
), IPI_DPC
);
604 // This routine sets the current thread in a swap busy state, which ensure that
605 // nobody else tries to swap it concurrently.
609 KiSetThreadSwapBusy(IN PKTHREAD Thread
)
611 /* Make sure nobody already set it */
612 ASSERT(Thread
->SwapBusy
== FALSE
);
614 /* Set it ourselves */
615 Thread
->SwapBusy
= TRUE
;
619 // This routine acquires the PRCB lock so that only one caller can touch
620 // volatile PRCB data.
622 // Since this is a simple optimized spin-lock, it must be be only acquired
623 // at dispatcher level or higher!
627 KiAcquirePrcbLock(IN PKPRCB Prcb
)
629 /* Make sure we're at a safe level to touch the PRCB lock */
630 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
632 /* Start acquire loop */
635 /* Acquire the lock and break out if we acquired it first */
636 if (!InterlockedExchange((PLONG
)&Prcb
->PrcbLock
, 1)) break;
638 /* Loop until the other CPU releases it */
641 /* Let the CPU know that this is a loop */
643 } while (Prcb
->PrcbLock
);
648 // This routine releases the PRCB lock so that other callers can touch
649 // volatile PRCB data.
651 // Since this is a simple optimized spin-lock, it must be be only acquired
652 // at dispatcher level or higher!
656 KiReleasePrcbLock(IN PKPRCB Prcb
)
658 /* Make sure it's acquired! */
659 ASSERT(Prcb
->PrcbLock
!= 0);
662 InterlockedAnd((PLONG
)&Prcb
->PrcbLock
, 0);
666 // This routine acquires the thread lock so that only one caller can touch
667 // volatile thread data.
669 // Since this is a simple optimized spin-lock, it must be be only acquired
670 // at dispatcher level or higher!
674 KiAcquireThreadLock(IN PKTHREAD Thread
)
676 /* Make sure we're at a safe level to touch the thread lock */
677 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
679 /* Start acquire loop */
682 /* Acquire the lock and break out if we acquired it first */
683 if (!InterlockedExchange((PLONG
)&Thread
->ThreadLock
, 1)) break;
685 /* Loop until the other CPU releases it */
688 /* Let the CPU know that this is a loop */
690 } while (Thread
->ThreadLock
);
695 // This routine releases the thread lock so that other callers can touch
696 // volatile thread data.
698 // Since this is a simple optimized spin-lock, it must be be only acquired
699 // at dispatcher level or higher!
703 KiReleaseThreadLock(IN PKTHREAD Thread
)
706 InterlockedAnd((PLONG
)&Thread
->ThreadLock
, 0);
711 KiTryThreadLock(IN PKTHREAD Thread
)
715 /* If the lock isn't acquired, return false */
716 if (!Thread
->ThreadLock
) return FALSE
;
718 /* Otherwise, try to acquire it and check the result */
720 Value
= InterlockedExchange((PLONG
)&Thread
->ThreadLock
, Value
);
722 /* Return the lock state */
723 return (Value
== TRUE
);
728 KiCheckDeferredReadyList(IN PKPRCB Prcb
)
730 /* Scan the deferred ready lists if required */
731 if (Prcb
->DeferredReadyListHead
.Next
) KiProcessDeferredReadyList(Prcb
);
736 KiRundownThread(IN PKTHREAD Thread
)
738 #if defined(_M_IX86) || defined(_M_AMD64)
740 ASSERTMSG("Not yet implemented\n", FALSE
);
746 KiRequestApcInterrupt(IN BOOLEAN NeedApc
,
749 /* Check if we need to request APC delivery */
752 /* Check if it's on another CPU */
753 if (KeGetPcr()->Number
!= Processor
)
755 /* Send an IPI to request delivery */
756 KiIpiSend(AFFINITY_MASK(Processor
), IPI_APC
);
760 /* Request a software interrupt */
761 HalRequestSoftwareInterrupt(APC_LEVEL
);
768 KiAcquireTimerLock(IN ULONG Hand
)
770 PKSPIN_LOCK_QUEUE LockQueue
;
772 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
774 /* Get the lock index */
775 LockIndex
= Hand
>> LOCK_QUEUE_TIMER_LOCK_SHIFT
;
776 LockIndex
&= (LOCK_QUEUE_TIMER_TABLE_LOCKS
- 1);
778 /* Now get the lock */
779 LockQueue
= &KeGetCurrentPrcb()->LockQueue
[LockQueueTimerTableLock
+ LockIndex
];
781 /* Acquire it and return */
782 KeAcquireQueuedSpinLockAtDpcLevel(LockQueue
);
788 KiReleaseTimerLock(IN PKSPIN_LOCK_QUEUE LockQueue
)
790 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL
);
792 /* Release the lock */
793 KeReleaseQueuedSpinLockFromDpcLevel(LockQueue
);
800 KiAcquireApcLock(IN PKTHREAD Thread
,
801 IN PKLOCK_QUEUE_HANDLE Handle
)
803 /* Acquire the lock and raise to synchronization level */
804 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread
->ApcQueueLock
, Handle
);
809 KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread
,
810 IN PKLOCK_QUEUE_HANDLE Handle
)
812 /* Acquire the lock */
813 KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread
->ApcQueueLock
, Handle
);
818 KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread
,
819 IN PKLOCK_QUEUE_HANDLE Handle
)
821 /* Acquire the lock */
822 KeAcquireInStackQueuedSpinLock(&Thread
->ApcQueueLock
, Handle
);
827 KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle
)
829 /* Release the lock */
830 KeReleaseInStackQueuedSpinLock(Handle
);
835 KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
837 /* Release the lock */
838 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
843 KiAcquireProcessLock(IN PKPROCESS Process
,
844 IN PKLOCK_QUEUE_HANDLE Handle
)
846 /* Acquire the lock and raise to synchronization level */
847 KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process
->ProcessLock
, Handle
);
852 KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle
)
854 /* Release the lock */
855 KeReleaseInStackQueuedSpinLock(Handle
);
860 KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle
)
862 /* Release the lock */
863 KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle
);
868 KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue
,
869 IN PKLOCK_QUEUE_HANDLE DeviceLock
)
871 /* Check if we were called from a threaded DPC */
872 if (KeGetCurrentPrcb()->DpcThreadActive
)
874 /* Lock the Queue, we're not at DPC level */
875 KeAcquireInStackQueuedSpinLock(&DeviceQueue
->Lock
, DeviceLock
);
879 /* We must be at DPC level, acquire the lock safely */
880 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
881 KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue
->Lock
,
888 KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock
)
890 /* Check if we were called from a threaded DPC */
891 if (KeGetCurrentPrcb()->DpcThreadActive
)
893 /* Unlock the Queue, we're not at DPC level */
894 KeReleaseInStackQueuedSpinLock(DeviceLock
);
898 /* We must be at DPC level, release the lock safely */
899 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
900 KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock
);
905 // Satisfies the wait of any dispatcher object
907 #define KiSatisfyObjectWait(Object, Thread) \
909 /* Special case for Mutants */ \
910 if ((Object)->Header.Type == MutantObject) \
912 /* Decrease the Signal State */ \
913 (Object)->Header.SignalState--; \
915 /* Check if it's now non-signaled */ \
916 if (!(Object)->Header.SignalState) \
918 /* Set the Owner Thread */ \
919 (Object)->OwnerThread = Thread; \
921 /* Disable APCs if needed */ \
922 Thread->KernelApcDisable = Thread->KernelApcDisable - \
923 (Object)->ApcDisable; \
925 /* Check if it's abandoned */ \
926 if ((Object)->Abandoned) \
929 (Object)->Abandoned = FALSE; \
931 /* Return Status */ \
932 Thread->WaitStatus = STATUS_ABANDONED; \
935 /* Insert it into the Mutant List */ \
936 InsertHeadList(Thread->MutantListHead.Blink, \
937 &(Object)->MutantListEntry); \
940 else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
941 EventSynchronizationObject) \
943 /* Synchronization Timers and Events just get un-signaled */ \
944 (Object)->Header.SignalState = 0; \
946 else if ((Object)->Header.Type == SemaphoreObject) \
948 /* These ones can have multiple states, so we only decrease it */ \
949 (Object)->Header.SignalState--; \
954 // Satisfies the wait of a mutant dispatcher object
956 #define KiSatisfyMutantWait(Object, Thread) \
958 /* Decrease the Signal State */ \
959 (Object)->Header.SignalState--; \
961 /* Check if it's now non-signaled */ \
962 if (!(Object)->Header.SignalState) \
964 /* Set the Owner Thread */ \
965 (Object)->OwnerThread = Thread; \
967 /* Disable APCs if needed */ \
968 Thread->KernelApcDisable = Thread->KernelApcDisable - \
969 (Object)->ApcDisable; \
971 /* Check if it's abandoned */ \
972 if ((Object)->Abandoned) \
975 (Object)->Abandoned = FALSE; \
977 /* Return Status */ \
978 Thread->WaitStatus = STATUS_ABANDONED; \
981 /* Insert it into the Mutant List */ \
982 InsertHeadList(Thread->MutantListHead.Blink, \
983 &(Object)->MutantListEntry); \
988 // Satisfies the wait of any nonmutant dispatcher object
990 #define KiSatisfyNonMutantWait(Object) \
992 if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
993 EventSynchronizationObject) \
995 /* Synchronization Timers and Events just get un-signaled */ \
996 (Object)->Header.SignalState = 0; \
998 else if ((Object)->Header.Type == SemaphoreObject) \
1000 /* These ones can have multiple states, so we only decrease it */ \
1001 (Object)->Header.SignalState--; \
1006 // Recalculates the due time
1010 KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime
,
1011 IN PLARGE_INTEGER DueTime
,
1012 IN OUT PLARGE_INTEGER NewDueTime
)
1014 /* Don't do anything for absolute waits */
1015 if (OriginalDueTime
->QuadPart
>= 0) return OriginalDueTime
;
1017 /* Otherwise, query the interrupt time and recalculate */
1018 NewDueTime
->QuadPart
= KeQueryInterruptTime();
1019 NewDueTime
->QuadPart
-= DueTime
->QuadPart
;
1024 // Determines whether a thread should be added to the wait list
1028 KiCheckThreadStackSwap(IN PKTHREAD Thread
,
1029 IN KPROCESSOR_MODE WaitMode
)
1031 /* Check the required conditions */
1032 if ((WaitMode
!= KernelMode
) &&
1033 (Thread
->EnableStackSwap
) &&
1034 (Thread
->Priority
>= (LOW_REALTIME_PRIORITY
+ 9)))
1036 /* We are go for swap */
1041 /* Don't swap the thread */
1047 // Adds a thread to the wait list
1049 #define KiAddThreadToWaitList(Thread, Swappable) \
1051 /* Make sure it's swappable */ \
1054 /* Insert it into the PRCB's List */ \
1055 InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
1056 &Thread->WaitListEntry); \
1061 // Checks if a wait in progress should be interrupted by APCs or an alertable
1066 KiCheckAlertability(IN PKTHREAD Thread
,
1067 IN BOOLEAN Alertable
,
1068 IN KPROCESSOR_MODE WaitMode
)
1070 /* Check if the wait is alertable */
1073 /* It is, first check if the thread is alerted in this mode */
1074 if (Thread
->Alerted
[WaitMode
])
1076 /* It is, so bail out of the wait */
1077 Thread
->Alerted
[WaitMode
] = FALSE
;
1078 return STATUS_ALERTED
;
1080 else if ((WaitMode
!= KernelMode
) &&
1081 (!IsListEmpty(&Thread
->ApcState
.ApcListHead
[UserMode
])))
1083 /* It's isn't, but this is a user wait with queued user APCs */
1084 Thread
->ApcState
.UserApcPending
= TRUE
;
1085 return STATUS_USER_APC
;
1087 else if (Thread
->Alerted
[KernelMode
])
1089 /* It isn't that either, but we're alered in kernel mode */
1090 Thread
->Alerted
[KernelMode
] = FALSE
;
1091 return STATUS_ALERTED
;
1094 else if ((WaitMode
!= KernelMode
) && (Thread
->ApcState
.UserApcPending
))
1096 /* Not alertable, but this is a user wait with pending user APCs */
1097 return STATUS_USER_APC
;
1100 /* Otherwise, we're fine */
1101 return STATUS_WAIT_0
;
1105 // Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
1106 // to remove timer entries
1107 // See Windows HPI blog for more information.
1110 KiRemoveEntryTimer(IN PKTIMER Timer
)
1113 PKTIMER_TABLE_ENTRY TableEntry
;
1115 /* Remove the timer from the timer list and check if it's empty */
1116 Hand
= Timer
->Header
.Hand
;
1117 if (RemoveEntryList(&Timer
->TimerListEntry
))
1119 /* Get the respective timer table entry */
1120 TableEntry
= &KiTimerTableListHead
[Hand
];
1121 if (&TableEntry
->Entry
== TableEntry
->Entry
.Flink
)
1123 /* Set the entry to an infinite absolute time */
1124 TableEntry
->Time
.HighPart
= 0xFFFFFFFF;
1128 /* Clear the list entries on dbg builds so we can tell the timer is gone */
1130 Timer
->TimerListEntry
.Flink
= NULL
;
1131 Timer
->TimerListEntry
.Blink
= NULL
;
1136 // Called by Wait and Queue code to insert a timer for dispatching.
1137 // Also called by KeSetTimerEx to insert a timer from the caller.
1141 KxInsertTimer(IN PKTIMER Timer
,
1144 PKSPIN_LOCK_QUEUE LockQueue
;
1146 /* Acquire the lock and release the dispatcher lock */
1147 LockQueue
= KiAcquireTimerLock(Hand
);
1148 KiReleaseDispatcherLockFromDpcLevel();
1150 /* Try to insert the timer */
1151 if (KiInsertTimerTable(Timer
, Hand
))
1154 KiCompleteTimer(Timer
, LockQueue
);
1158 /* Do nothing, just release the lock */
1159 KiReleaseTimerLock(LockQueue
);
1164 // Called by KeSetTimerEx and KiInsertTreeTimer to calculate Due Time
1165 // See the Windows HPI Blog for more information
1169 KiComputeDueTime(IN PKTIMER Timer
,
1170 IN LARGE_INTEGER DueTime
,
1173 LARGE_INTEGER InterruptTime
, SystemTime
, DifferenceTime
;
1175 /* Convert to relative time if needed */
1176 Timer
->Header
.Absolute
= FALSE
;
1177 if (DueTime
.HighPart
>= 0)
1179 /* Get System Time */
1180 KeQuerySystemTime(&SystemTime
);
1182 /* Do the conversion */
1183 DifferenceTime
.QuadPart
= SystemTime
.QuadPart
- DueTime
.QuadPart
;
1185 /* Make sure it hasn't already expired */
1186 Timer
->Header
.Absolute
= TRUE
;
1187 if (DifferenceTime
.HighPart
>= 0)
1189 /* Cancel everything */
1190 Timer
->Header
.SignalState
= TRUE
;
1191 Timer
->Header
.Hand
= 0;
1192 Timer
->DueTime
.QuadPart
= 0;
1197 /* Set the time as Absolute */
1198 DueTime
= DifferenceTime
;
1201 /* Get the Interrupt Time */
1202 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1204 /* Recalculate due time */
1205 Timer
->DueTime
.QuadPart
= InterruptTime
.QuadPart
- DueTime
.QuadPart
;
1207 /* Get the handle */
1208 *Hand
= KiComputeTimerTableIndex(Timer
->DueTime
.QuadPart
);
1209 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1210 Timer
->Header
.Inserted
= TRUE
;
1215 // Called from Unlink and Queue Insert Code.
1216 // Also called by timer code when canceling an inserted timer.
1217 // Removes a timer from it's tree.
1221 KxRemoveTreeTimer(IN PKTIMER Timer
)
1223 ULONG Hand
= Timer
->Header
.Hand
;
1224 PKSPIN_LOCK_QUEUE LockQueue
;
1225 PKTIMER_TABLE_ENTRY TimerEntry
;
1227 /* Acquire timer lock */
1228 LockQueue
= KiAcquireTimerLock(Hand
);
1230 /* Set the timer as non-inserted */
1231 Timer
->Header
.Inserted
= FALSE
;
1233 /* Remove it from the timer list */
1234 if (RemoveEntryList(&Timer
->TimerListEntry
))
1236 /* Get the entry and check if it's empty */
1237 TimerEntry
= &KiTimerTableListHead
[Hand
];
1238 if (IsListEmpty(&TimerEntry
->Entry
))
1240 /* Clear the time then */
1241 TimerEntry
->Time
.HighPart
= 0xFFFFFFFF;
1245 /* Release the timer lock */
1246 KiReleaseTimerLock(LockQueue
);
1251 KxSetTimerForThreadWait(IN PKTIMER Timer
,
1252 IN LARGE_INTEGER Interval
,
1256 LARGE_INTEGER InterruptTime
, SystemTime
, TimeDifference
;
1258 /* Check the timer's interval to see if it's absolute */
1259 Timer
->Header
.Absolute
= FALSE
;
1260 if (Interval
.HighPart
>= 0)
1262 /* Get the system time and calculate the relative time */
1263 KeQuerySystemTime(&SystemTime
);
1264 TimeDifference
.QuadPart
= SystemTime
.QuadPart
- Interval
.QuadPart
;
1265 Timer
->Header
.Absolute
= TRUE
;
1267 /* Check if we've already expired */
1268 if (TimeDifference
.HighPart
>= 0)
1270 /* Reset everything */
1271 Timer
->DueTime
.QuadPart
= 0;
1273 Timer
->Header
.Hand
= 0;
1278 /* Update the interval */
1279 Interval
= TimeDifference
;
1283 /* Calculate the due time */
1284 InterruptTime
.QuadPart
= KeQueryInterruptTime();
1285 DueTime
= InterruptTime
.QuadPart
- Interval
.QuadPart
;
1286 Timer
->DueTime
.QuadPart
= DueTime
;
1288 /* Calculate the timer handle */
1289 *Hand
= KiComputeTimerTableIndex(DueTime
);
1290 Timer
->Header
.Hand
= (UCHAR
)*Hand
;
1293 #define KxDelayThreadWait() \
1295 /* Setup the Wait Block */ \
1296 Thread->WaitBlockList = TimerBlock; \
1298 /* Setup the timer */ \
1299 KxSetTimerForThreadWait(Timer, *Interval, &Hand); \
1301 /* Save the due time for the caller */ \
1302 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1304 /* Link the timer to this Wait Block */ \
1305 TimerBlock->NextWaitBlock = TimerBlock; \
1306 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1307 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1309 /* Clear wait status */ \
1310 Thread->WaitStatus = STATUS_SUCCESS; \
1312 /* Setup wait fields */ \
1313 Thread->Alertable = Alertable; \
1314 Thread->WaitReason = DelayExecution; \
1315 Thread->WaitMode = WaitMode; \
1317 /* Check if we can swap the thread's stack */ \
1318 Thread->WaitListEntry.Flink = NULL; \
1319 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1321 /* Set the wait time */ \
1322 Thread->WaitTime = KeTickCount.LowPart;
1324 #define KxMultiThreadWait() \
1325 /* Link wait block array to the thread */ \
1326 Thread->WaitBlockList = WaitBlockArray; \
1328 /* Reset the index */ \
1331 /* Loop wait blocks */ \
1334 /* Fill out the wait block */ \
1335 WaitBlock = &WaitBlockArray[Index]; \
1336 WaitBlock->Object = Object[Index]; \
1337 WaitBlock->WaitKey = (USHORT)Index; \
1338 WaitBlock->WaitType = WaitType; \
1339 WaitBlock->Thread = Thread; \
1341 /* Link to next block */ \
1342 WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1]; \
1344 } while (Index < Count); \
1346 /* Link the last block */ \
1347 WaitBlock->NextWaitBlock = WaitBlockArray; \
1349 /* Set default wait status */ \
1350 Thread->WaitStatus = STATUS_WAIT_0; \
1352 /* Check if we have a timer */ \
1355 /* Link to the block */ \
1356 TimerBlock->NextWaitBlock = WaitBlockArray; \
1358 /* Setup the timer */ \
1359 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1361 /* Save the due time for the caller */ \
1362 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1364 /* Initialize the list */ \
1365 InitializeListHead(&Timer->Header.WaitListHead); \
1368 /* Set wait settings */ \
1369 Thread->Alertable = Alertable; \
1370 Thread->WaitMode = WaitMode; \
1371 Thread->WaitReason = WaitReason; \
1373 /* Check if we can swap the thread's stack */ \
1374 Thread->WaitListEntry.Flink = NULL; \
1375 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1377 /* Set the wait time */ \
1378 Thread->WaitTime = KeTickCount.LowPart;
1380 #define KxSingleThreadWait() \
1381 /* Setup the Wait Block */ \
1382 Thread->WaitBlockList = WaitBlock; \
1383 WaitBlock->WaitKey = STATUS_SUCCESS; \
1384 WaitBlock->Object = Object; \
1385 WaitBlock->WaitType = WaitAny; \
1387 /* Clear wait status */ \
1388 Thread->WaitStatus = STATUS_SUCCESS; \
1390 /* Check if we have a timer */ \
1393 /* Setup the timer */ \
1394 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1396 /* Save the due time for the caller */ \
1397 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1399 /* Pointer to timer block */ \
1400 WaitBlock->NextWaitBlock = TimerBlock; \
1401 TimerBlock->NextWaitBlock = WaitBlock; \
1403 /* Link the timer to this Wait Block */ \
1404 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1405 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1409 /* No timer block, just ourselves */ \
1410 WaitBlock->NextWaitBlock = WaitBlock; \
1413 /* Set wait settings */ \
1414 Thread->Alertable = Alertable; \
1415 Thread->WaitMode = WaitMode; \
1416 Thread->WaitReason = WaitReason; \
1418 /* Check if we can swap the thread's stack */ \
1419 Thread->WaitListEntry.Flink = NULL; \
1420 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1422 /* Set the wait time */ \
1423 Thread->WaitTime = KeTickCount.LowPart;
1425 #define KxQueueThreadWait() \
1426 /* Setup the Wait Block */ \
1427 Thread->WaitBlockList = WaitBlock; \
1428 WaitBlock->WaitKey = STATUS_SUCCESS; \
1429 WaitBlock->Object = Queue; \
1430 WaitBlock->WaitType = WaitAny; \
1431 WaitBlock->Thread = Thread; \
1433 /* Clear wait status */ \
1434 Thread->WaitStatus = STATUS_SUCCESS; \
1436 /* Check if we have a timer */ \
1439 /* Setup the timer */ \
1440 KxSetTimerForThreadWait(Timer, *Timeout, &Hand); \
1442 /* Save the due time for the caller */ \
1443 DueTime.QuadPart = Timer->DueTime.QuadPart; \
1445 /* Pointer to timer block */ \
1446 WaitBlock->NextWaitBlock = TimerBlock; \
1447 TimerBlock->NextWaitBlock = WaitBlock; \
1449 /* Link the timer to this Wait Block */ \
1450 Timer->Header.WaitListHead.Flink = &TimerBlock->WaitListEntry; \
1451 Timer->Header.WaitListHead.Blink = &TimerBlock->WaitListEntry; \
1455 /* No timer block, just ourselves */ \
1456 WaitBlock->NextWaitBlock = WaitBlock; \
1459 /* Set wait settings */ \
1460 Thread->Alertable = FALSE; \
1461 Thread->WaitMode = WaitMode; \
1462 Thread->WaitReason = WrQueue; \
1464 /* Check if we can swap the thread's stack */ \
1465 Thread->WaitListEntry.Flink = NULL; \
1466 Swappable = KiCheckThreadStackSwap(Thread, WaitMode); \
1468 /* Set the wait time */ \
1469 Thread->WaitTime = KeTickCount.LowPart;
1476 KxUnwaitThread(IN DISPATCHER_HEADER
*Object
,
1477 IN KPRIORITY Increment
)
1479 PLIST_ENTRY WaitEntry
, WaitList
;
1480 PKWAIT_BLOCK WaitBlock
;
1481 PKTHREAD WaitThread
;
1484 /* Loop the Wait Entries */
1485 WaitList
= &Object
->WaitListHead
;
1486 ASSERT(IsListEmpty(&Object
->WaitListHead
) == FALSE
);
1487 WaitEntry
= WaitList
->Flink
;
1490 /* Get the current wait block */
1491 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1493 /* Get the waiting thread */
1494 WaitThread
= WaitBlock
->Thread
;
1496 /* Check the current Wait Mode */
1497 if (WaitBlock
->WaitType
== WaitAny
)
1499 /* Use the actual wait key */
1500 WaitKey
= WaitBlock
->WaitKey
;
1504 /* Otherwise, use STATUS_KERNEL_APC */
1505 WaitKey
= STATUS_KERNEL_APC
;
1508 /* Unwait the thread */
1509 KiUnwaitThread(WaitThread
, WaitKey
, Increment
);
1512 WaitEntry
= WaitList
->Flink
;
1513 } while (WaitEntry
!= WaitList
);
1517 // Unwaits a Thread waiting on an event
1521 KxUnwaitThreadForEvent(IN PKEVENT Event
,
1522 IN KPRIORITY Increment
)
1524 PLIST_ENTRY WaitEntry
, WaitList
;
1525 PKWAIT_BLOCK WaitBlock
;
1526 PKTHREAD WaitThread
;
1528 /* Loop the Wait Entries */
1529 WaitList
= &Event
->Header
.WaitListHead
;
1530 ASSERT(IsListEmpty(&Event
->Header
.WaitListHead
) == FALSE
);
1531 WaitEntry
= WaitList
->Flink
;
1534 /* Get the current wait block */
1535 WaitBlock
= CONTAINING_RECORD(WaitEntry
, KWAIT_BLOCK
, WaitListEntry
);
1537 /* Get the waiting thread */
1538 WaitThread
= WaitBlock
->Thread
;
1540 /* Check the current Wait Mode */
1541 if (WaitBlock
->WaitType
== WaitAny
)
1544 Event
->Header
.SignalState
= 0;
1546 /* Un-signal the event and unwait the thread */
1547 KiUnwaitThread(WaitThread
, WaitBlock
->WaitKey
, Increment
);
1551 /* Unwait the thread with STATUS_KERNEL_APC */
1552 KiUnwaitThread(WaitThread
, STATUS_KERNEL_APC
, Increment
);
1555 WaitEntry
= WaitList
->Flink
;
1556 } while (WaitEntry
!= WaitList
);
1560 // This routine queues a thread that is ready on the PRCB's ready lists.
1561 // If this thread cannot currently run on this CPU, then the thread is
1562 // added to the deferred ready list instead.
1564 // This routine must be entered with the PRCB lock held and it will exit
1565 // with the PRCB lock released!
1569 KxQueueReadyThread(IN PKTHREAD Thread
,
1576 ASSERT(Prcb
== KeGetCurrentPrcb());
1577 ASSERT(Thread
->State
== Running
);
1578 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1580 /* Check if this thread is allowed to run in this CPU */
1582 if ((Thread
->Affinity
) & (Prcb
->SetMember
))
1587 /* Set thread ready for execution */
1588 Thread
->State
= Ready
;
1590 /* Save current priority and if someone had pre-empted it */
1591 Priority
= Thread
->Priority
;
1592 Preempted
= Thread
->Preempted
;
1594 /* We're not pre-empting now, and set the wait time */
1595 Thread
->Preempted
= FALSE
;
1596 Thread
->WaitTime
= KeTickCount
.LowPart
;
1599 ASSERT((Priority
>= 0) && (Priority
<= HIGH_PRIORITY
));
1601 /* Insert this thread in the appropriate order */
1602 Preempted
? InsertHeadList(&Prcb
->DispatcherReadyListHead
[Priority
],
1603 &Thread
->WaitListEntry
) :
1604 InsertTailList(&Prcb
->DispatcherReadyListHead
[Priority
],
1605 &Thread
->WaitListEntry
);
1607 /* Update the ready summary */
1608 Prcb
->ReadySummary
|= PRIORITY_MASK(Priority
);
1611 ASSERT(Priority
== Thread
->Priority
);
1613 /* Release the PRCB lock */
1614 KiReleasePrcbLock(Prcb
);
1618 /* Otherwise, prepare this thread to be deferred */
1619 Thread
->State
= DeferredReady
;
1620 Thread
->DeferredProcessor
= Prcb
->Number
;
1622 /* Release the lock and defer scheduling */
1623 KiReleasePrcbLock(Prcb
);
1624 KiDeferredReadyThread(Thread
);
1629 // This routine scans for an appropriate ready thread to select at the
1630 // given priority and for the given CPU.
1634 KiSelectReadyThread(IN KPRIORITY Priority
,
1639 PLIST_ENTRY ListEntry
;
1640 PKTHREAD Thread
= NULL
;
1642 /* Save the current mask and get the priority set for the CPU */
1643 PrioritySet
= Prcb
->ReadySummary
>> Priority
;
1644 if (!PrioritySet
) goto Quickie
;
1646 /* Get the highest priority possible */
1647 BitScanReverse((PULONG
)&HighPriority
, PrioritySet
);
1648 ASSERT((PrioritySet
& PRIORITY_MASK(HighPriority
)) != 0);
1649 HighPriority
+= Priority
;
1651 /* Make sure the list isn't empty at the highest priority */
1652 ASSERT(IsListEmpty(&Prcb
->DispatcherReadyListHead
[HighPriority
]) == FALSE
);
1654 /* Get the first thread on the list */
1655 ListEntry
= Prcb
->DispatcherReadyListHead
[HighPriority
].Flink
;
1656 Thread
= CONTAINING_RECORD(ListEntry
, KTHREAD
, WaitListEntry
);
1658 /* Make sure this thread is here for a reason */
1659 ASSERT(HighPriority
== Thread
->Priority
);
1660 ASSERT(Thread
->Affinity
& AFFINITY_MASK(Prcb
->Number
));
1661 ASSERT(Thread
->NextProcessor
== Prcb
->Number
);
1663 /* Remove it from the list */
1664 if (RemoveEntryList(&Thread
->WaitListEntry
))
1666 /* The list is empty now, reset the ready summary */
1667 Prcb
->ReadySummary
^= PRIORITY_MASK(HighPriority
);
1670 /* Sanity check and return the thread */
1672 ASSERT((Thread
== NULL
) ||
1673 (Thread
->BasePriority
== 0) ||
1674 (Thread
->Priority
!= 0));
1679 // This routine computes the new priority for a thread. It is only valid for
1680 // threads with priorities in the dynamic priority range.
1684 KiComputeNewPriority(IN PKTHREAD Thread
,
1685 IN SCHAR Adjustment
)
1689 /* Priority sanity checks */
1690 ASSERT((Thread
->PriorityDecrement
>= 0) &&
1691 (Thread
->PriorityDecrement
<= Thread
->Priority
));
1692 ASSERT((Thread
->Priority
< LOW_REALTIME_PRIORITY
) ?
1693 TRUE
: (Thread
->PriorityDecrement
== 0));
1695 /* Get the current priority */
1696 Priority
= Thread
->Priority
;
1697 if (Priority
< LOW_REALTIME_PRIORITY
)
1699 /* Decrease priority by the priority decrement */
1700 Priority
-= (Thread
->PriorityDecrement
+ Adjustment
);
1702 /* Don't go out of bounds */
1703 if (Priority
< Thread
->BasePriority
) Priority
= Thread
->BasePriority
;
1705 /* Reset the priority decrement */
1706 Thread
->PriorityDecrement
= 0;
1710 ASSERT((Thread
->BasePriority
== 0) || (Priority
!= 0));
1712 /* Return the new priority */