-/*\r
-* PROJECT: ReactOS Kernel\r
-* LICENSE: GPL - See COPYING in the top level directory\r
-* FILE: ntoskrnl/include/ke_x.h\r
-* PURPOSE: Internal Inlined Functions for the Kernel\r
-* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)\r
-*/\r
-\r
-//\r
-// Enters a Guarded Region\r
-//\r
-#define KeEnterGuardedRegion() \\r
-{ \\r
- PKTHREAD Thread = KeGetCurrentThread(); \\r
- \\r
- /* Sanity checks */ \\r
- ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \\r
- ASSERT(Thread == KeGetCurrentThread()); \\r
- ASSERT((Thread->SpecialApcDisable <= 0) && \\r
- (Thread->SpecialApcDisable != -32768)); \\r
- \\r
- /* Disable Special APCs */ \\r
- Thread->SpecialApcDisable--; \\r
-}\r
-\r
-//\r
-// Leaves a Guarded Region\r
-//\r
-#define KeLeaveGuardedRegion() \\r
-{ \\r
- PKTHREAD Thread = KeGetCurrentThread(); \\r
- \\r
- /* Sanity checks */ \\r
- ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \\r
- ASSERT(Thread == KeGetCurrentThread()); \\r
- ASSERT(Thread->SpecialApcDisable < 0); \\r
- \\r
- /* Leave region and check if APCs are OK now */ \\r
- if (!(++Thread->SpecialApcDisable)) \\r
- { \\r
- /* Check for Kernel APCs on the list */ \\r
- if (!IsListEmpty(&Thread->ApcState. \\r
- ApcListHead[KernelMode])) \\r
- { \\r
- /* Check for APC Delivery */ \\r
- KiCheckForKernelApcDelivery(); \\r
- } \\r
- } \\r
-}\r
-\r
-//\r
-// TODO: Guarded Mutex Routines\r
-//\r
-\r
-//\r
-// Enters a Critical Region\r
-//\r
-#define KeEnterCriticalRegion() \\r
-{ \\r
- PKTHREAD Thread = KeGetCurrentThread(); \\r
- if (Thread) \\r
- { \\r
- /* Sanity checks */ \\r
- ASSERT(Thread == KeGetCurrentThread()); \\r
- ASSERT((Thread->KernelApcDisable <= 0) && \\r
- (Thread->KernelApcDisable != -32768)); \\r
- \\r
- /* Disable Kernel APCs */ \\r
- Thread->KernelApcDisable--; \\r
- } \\r
-}\r
-\r
-//\r
-// Leaves a Critical Region\r
-//\r
-#define KeLeaveCriticalRegion() \\r
-{ \\r
- PKTHREAD Thread = KeGetCurrentThread(); \\r
- if (Thread) \\r
- { \\r
- /* Sanity checks */ \\r
- ASSERT(Thread == KeGetCurrentThread()); \\r
- ASSERT(Thread->KernelApcDisable < 0); \\r
- \\r
- /* Enable Kernel APCs */ \\r
- Thread->KernelApcDisable++; \\r
- \\r
- /* Check if Kernel APCs are now enabled */ \\r
- if (!(Thread->KernelApcDisable)) \\r
- { \\r
- /* Check if we need to request an APC Delivery */ \\r
- if (!(IsListEmpty(&Thread->ApcState.ApcListHead[KernelMode])) && \\r
- !(Thread->KernelApcDisable)) \\r
- { \\r
- /* Check for the right environment */ \\r
- KiCheckForKernelApcDelivery(); \\r
- } \\r
- } \\r
- } \\r
-}\r
-\r
-//\r
-// Satisfies the wait of any dispatcher object\r
-//\r
-#define KiSatisfyObjectWait(Object, Thread) \\r
-{ \\r
- /* Special case for Mutants */ \\r
- if ((Object)->Header.Type == MutantObject) \\r
- { \\r
- /* Decrease the Signal State */ \\r
- (Object)->Header.SignalState--; \\r
- \\r
- /* Check if it's now non-signaled */ \\r
- if (!(Object)->Header.SignalState) \\r
- { \\r
- /* Set the Owner Thread */ \\r
- (Object)->OwnerThread = Thread; \\r
- \\r
- /* Disable APCs if needed */ \\r
- Thread->KernelApcDisable -= (Object)->ApcDisable; \\r
- \\r
- /* Check if it's abandoned */ \\r
- if ((Object)->Abandoned) \\r
- { \\r
- /* Unabandon it */ \\r
- (Object)->Abandoned = FALSE; \\r
- \\r
- /* Return Status */ \\r
- Thread->WaitStatus = STATUS_ABANDONED; \\r
- } \\r
- \\r
- /* Insert it into the Mutant List */ \\r
- InsertHeadList(Thread->MutantListHead.Blink, \\r
- &(Object)->MutantListEntry); \\r
- } \\r
- } \\r
- else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \\r
- EventSynchronizationObject) \\r
- { \\r
- /* Synchronization Timers and Events just get un-signaled */ \\r
- (Object)->Header.SignalState = 0; \\r
- } \\r
- else if ((Object)->Header.Type == SemaphoreObject) \\r
- { \\r
- /* These ones can have multiple states, so we only decrease it */ \\r
- (Object)->Header.SignalState--; \\r
- } \\r
-}\r
-\r
-//\r
-// Satisfies the wait of a mutant dispatcher object\r
-//\r
-#define KiSatisfyMutantWait(Object, Thread) \\r
-{ \\r
- /* Decrease the Signal State */ \\r
- (Object)->Header.SignalState--; \\r
- \\r
- /* Check if it's now non-signaled */ \\r
- if (!(Object)->Header.SignalState) \\r
- { \\r
- /* Set the Owner Thread */ \\r
- (Object)->OwnerThread = Thread; \\r
- \\r
- /* Disable APCs if needed */ \\r
- Thread->KernelApcDisable -= (Object)->ApcDisable; \\r
- \\r
- /* Check if it's abandoned */ \\r
- if ((Object)->Abandoned) \\r
- { \\r
- /* Unabandon it */ \\r
- (Object)->Abandoned = FALSE; \\r
- \\r
- /* Return Status */ \\r
- Thread->WaitStatus = STATUS_ABANDONED; \\r
- } \\r
- \\r
- /* Insert it into the Mutant List */ \\r
- InsertHeadList(Thread->MutantListHead.Blink, \\r
- &(Object)->MutantListEntry); \\r
- } \\r
-}\r
-\r
-//\r
-// Satisfies the wait of any nonmutant dispatcher object\r
-//\r
-#define KiSatisfyNonMutantWait(Object, Thread) \\r
-{ \\r
- if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \\r
- EventSynchronizationObject) \\r
- { \\r
- /* Synchronization Timers and Events just get un-signaled */ \\r
- (Object)->Header.SignalState = 0; \\r
- } \\r
- else if ((Object)->Header.Type == SemaphoreObject) \\r
- { \\r
- /* These ones can have multiple states, so we only decrease it */ \\r
- (Object)->Header.SignalState--; \\r
- } \\r
-}\r
-\r
-//\r
-// Recalculates the due time\r
-//\r
-PLARGE_INTEGER\r
-FORCEINLINE\r
-KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,\r
- IN PLARGE_INTEGER DueTime,\r
- IN OUT PLARGE_INTEGER NewDueTime)\r
-{\r
- /* Don't do anything for absolute waits */\r
- if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;\r
-\r
- /* Otherwise, query the interrupt time and recalculate */\r
- NewDueTime->QuadPart = KeQueryInterruptTime();\r
- NewDueTime->QuadPart -= DueTime->QuadPart;\r
- return NewDueTime;\r
-}\r
-\r
-//\r
-// Determines wether a thread should be added to the wait list\r
-//\r
-#define KiCheckThreadStackSwap(WaitMode, Thread, Swappable) \\r
-{ \\r
- /* Check the required conditions */ \\r
- if ((WaitMode != KernelMode) && \\r
- (Thread->EnableStackSwap) && \\r
- (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9))) \\r
- { \\r
- /* We are go for swap */ \\r
- Swappable = TRUE; \\r
- } \\r
- else \\r
- { \\r
- /* Don't swap the thread */ \\r
- Swappable = FALSE; \\r
- } \\r
-}\r
-\r
-//\r
-// Adds a thread to the wait list\r
-//\r
-#define KiAddThreadToWaitList(Thread, Swappable) \\r
-{ \\r
- /* Make sure it's swappable */ \\r
- if (Swappable) \\r
- { \\r
- /* Insert it into the PRCB's List */ \\r
- InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \\r
- &Thread->WaitListEntry); \\r
- } \\r
-}\r
-\r
-//\r
-// Rules for checking alertability:\r
-// - For Alertable waits ONLY:\r
-// * We don't wait and return STATUS_ALERTED if the thread is alerted\r
-// in EITHER the specified wait mode OR in Kernel Mode.\r
-// - For BOTH Alertable AND Non-Alertable waits:\r
-// * We don't want and return STATUS_USER_APC if the User Mode APC list\r
-// is not empty AND the wait mode is User Mode.\r
-//\r
-#define KiCheckAlertability() \\r
-{ \\r
- if (Alertable) \\r
- { \\r
- if (CurrentThread->Alerted[(int)WaitMode]) \\r
- { \\r
- CurrentThread->Alerted[(int)WaitMode] = FALSE; \\r
- WaitStatus = STATUS_ALERTED; \\r
- break; \\r
- } \\r
- else if ((WaitMode != KernelMode) && \\r
- (!IsListEmpty(&CurrentThread-> \\r
- ApcState.ApcListHead[UserMode]))) \\r
- { \\r
- CurrentThread->ApcState.UserApcPending = TRUE; \\r
- WaitStatus = STATUS_USER_APC; \\r
- break; \\r
- } \\r
- else if (CurrentThread->Alerted[KernelMode]) \\r
- { \\r
- CurrentThread->Alerted[KernelMode] = FALSE; \\r
- WaitStatus = STATUS_ALERTED; \\r
- break; \\r
- } \\r
- } \\r
- else if ((WaitMode != KernelMode) && \\r
- (CurrentThread->ApcState.UserApcPending)) \\r
- { \\r
- WaitStatus = STATUS_USER_APC; \\r
- break; \\r
- } \\r
-}\r
-\r
-//\r
-// Thread Scheduling Routines\r
-//\r
-#ifndef _CONFIG_SMP\r
-KIRQL\r
-FORCEINLINE\r
-KiAcquireDispatcherLock(VOID)\r
-{\r
- /* Raise to DPC level */\r
- return KeRaiseIrqlToDpcLevel();\r
-}\r
-\r
-VOID\r
-FORCEINLINE\r
-KiReleaseDispatcherLock(IN KIRQL OldIrql)\r
-{\r
- /* Just exit the dispatcher */\r
- KiExitDispatcher(OldIrql);\r
-}\r
-\r
-VOID\r
-FORCEINLINE\r
-KiAcquireDispatcherLockAtDpcLevel(VOID)\r
-{\r
- /* This is a no-op at DPC Level for UP systems */\r
- return;\r
-}\r
-\r
-VOID\r
-FORCEINLINE\r
-KiReleaseDispatcherLockFromDpcLevel(VOID)\r
-{\r
- /* This is a no-op at DPC Level for UP systems */\r
- return;\r
-}\r
-\r
-//\r
-// This routine makes the thread deferred ready on the boot CPU.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiInsertDeferredReadyList(IN PKTHREAD Thread)\r
-{\r
- /* Set the thread to deferred state and boot CPU */\r
- Thread->State = DeferredReady;\r
- Thread->DeferredProcessor = 0;\r
-\r
- /* Make the thread ready immediately */\r
- KiDeferredReadyThread(Thread);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiRescheduleThread(IN BOOLEAN NewThread,\r
- IN ULONG Cpu)\r
-{\r
- /* This is meaningless on UP systems */\r
- UNREFERENCED_PARAMETER(NewThread);\r
- UNREFERENCED_PARAMETER(Cpu);\r
-}\r
-\r
-//\r
-// This routine protects against multiple CPU acquires, it's meaningless on UP.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiSetThreadSwapBusy(IN PKTHREAD Thread)\r
-{\r
- UNREFERENCED_PARAMETER(Thread);\r
-}\r
-\r
-//\r
-// This routine protects against multiple CPU acquires, it's meaningless on UP.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiAcquirePrcbLock(IN PKPRCB Prcb)\r
-{\r
- UNREFERENCED_PARAMETER(Prcb);\r
-}\r
-\r
-//\r
-// This routine protects against multiple CPU acquires, it's meaningless on UP.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiReleasePrcbLock(IN PKPRCB Prcb)\r
-{\r
- UNREFERENCED_PARAMETER(Prcb);\r
-}\r
-\r
-//\r
-// This routine protects against multiple CPU acquires, it's meaningless on UP.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiAcquireThreadLock(IN PKTHREAD Thread)\r
-{\r
- UNREFERENCED_PARAMETER(Thread);\r
-}\r
-\r
-//\r
-// This routine protects against multiple CPU acquires, it's meaningless on UP.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiReleaseThreadLock(IN PKTHREAD Thread)\r
-{\r
- UNREFERENCED_PARAMETER(Thread);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiCheckDeferredReadyList(IN PKPRCB Prcb)\r
-{\r
- /* There are no deferred ready lists on UP systems */\r
- UNREFERENCED_PARAMETER(Prcb);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiRundownThread(IN PKTHREAD Thread)\r
-{\r
- /* Check if this is the NPX Thread */\r
- if (KeGetCurrentPrcb()->NpxThread == Thread)\r
- {\r
- /* Clear it */\r
- KeGetCurrentPrcb()->NpxThread = NULL;\r
-#ifdef __GNUC__\r
- __asm__("fninit\n\t");\r
-#else\r
- __asm fninit;\r
-#endif\r
- }\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiRequestApcInterrupt(IN BOOLEAN NeedApc,\r
- IN UCHAR Processor)\r
-{\r
- /* We deliver instantly on UP */\r
- UNREFERENCED_PARAMETER(NeedApc);\r
- UNREFERENCED_PARAMETER(Processor);\r
-}\r
-\r
-#else\r
-\r
-KIRQL\r
-FORCEINLINE\r
-KiAcquireDispatcherLock(VOID)\r
-{\r
- /* Raise to synchronization level and acquire the dispatcher lock */\r
- return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock);\r
-}\r
-\r
-VOID\r
-FORCEINLINE\r
-KiReleaseDispatcherLock(IN KIRQL OldIrql)\r
-{\r
- /* First release the lock */\r
- KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->\r
- LockQueue[LockQueueDispatcherLock]);\r
-\r
- /* Then exit the dispatcher */\r
- KiExitDispatcher(OldIrql);\r
-}\r
-\r
-//\r
-// This routine inserts a thread into the deferred ready list of the given CPU\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiInsertDeferredReadyList(IN PKTHREAD Thread)\r
-{\r
- PKPRCB Prcb = KeGetCurrentPrcb();\r
-\r
- /* Set the thread to deferred state and CPU */\r
- Thread->State = DeferredReady;\r
- Thread->DeferredProcessor = Prcb->Number;\r
-\r
- /* Add it on the list */\r
- PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiRescheduleThread(IN BOOLEAN NewThread,\r
- IN ULONG Cpu)\r
-{\r
- /* Check if a new thread needs to be scheduled on a different CPU */\r
- if ((NewThread) && !(KeGetPcr()->Number == Cpu))\r
- {\r
- /* Send an IPI to request delivery */\r
- KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);\r
- }\r
-}\r
-\r
-//\r
-// This routine sets the current thread in a swap busy state, which ensure that\r
-// nobody else tries to swap it concurrently.\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiSetThreadSwapBusy(IN PKTHREAD Thread)\r
-{\r
- /* Make sure nobody already set it */\r
- ASSERT(Thread->SwapBusy == FALSE);\r
-\r
- /* Set it ourselves */\r
- Thread->SwapBusy = TRUE;\r
-}\r
-\r
-//\r
-// This routine acquires the PRCB lock so that only one caller can touch\r
-// volatile PRCB data.\r
-//\r
-// Since this is a simple optimized spin-lock, it must be be only acquired\r
-// at dispatcher level or higher!\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiAcquirePrcbLock(IN PKPRCB Prcb)\r
-{\r
- /* Make sure we're at a safe level to touch the PRCB lock */\r
- ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);\r
-\r
- /* Start acquire loop */\r
- for (;;)\r
- {\r
- /* Acquire the lock and break out if we acquired it first */\r
- if (!InterlockedExchange(&Prcb->PrcbLock, 1)) break;\r
-\r
- /* Loop until the other CPU releases it */\r
- do\r
- {\r
- /* Let the CPU know that this is a loop */\r
- YieldProcessor();\r
- } while (Prcb->PrcbLock);\r
- }\r
-}\r
-\r
-//\r
-// This routine releases the PRCB lock so that other callers can touch\r
-// volatile PRCB data.\r
-//\r
-// Since this is a simple optimized spin-lock, it must be be only acquired\r
-// at dispatcher level or higher!\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiReleasePrcbLock(IN PKPRCB Prcb)\r
-{\r
- /* Make sure it's acquired! */\r
- ASSERT(Prcb->PrcbLock != 0);\r
-\r
- /* Release it */\r
- InterlockedAnd(&Prcb->PrcbLock, 0);\r
-}\r
-\r
-//\r
-// This routine acquires the thread lock so that only one caller can touch\r
-// volatile thread data.\r
-//\r
-// Since this is a simple optimized spin-lock, it must be be only acquired\r
-// at dispatcher level or higher!\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiAcquireThreadLock(IN PKTHREAD Thread)\r
-{\r
- /* Make sure we're at a safe level to touch the thread lock */\r
- ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);\r
-\r
- /* Start acquire loop */\r
- for (;;)\r
- {\r
- /* Acquire the lock and break out if we acquired it first */\r
- if (!InterlockedExchange(&Thread->ThreadLock, 1)) break;\r
-\r
- /* Loop until the other CPU releases it */\r
- do\r
- {\r
- /* Let the CPU know that this is a loop */\r
- YieldProcessor();\r
- } while (Thread->ThreadLock);\r
- }\r
-}\r
-\r
-//\r
-// This routine releases the thread lock so that other callers can touch\r
-// volatile thread data.\r
-//\r
-// Since this is a simple optimized spin-lock, it must be be only acquired\r
-// at dispatcher level or higher!\r
-//\r
-FORCEINLINE\r
-VOID\r
-KiReleaseThreadLock(IN PKTHREAD Thread)\r
-{\r
- /* Release it */\r
- InterlockedAnd(&Thread->ThreadLock, 0);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiCheckDeferredReadyList(IN PKPRCB Prcb)\r
-{\r
- /* Scan the deferred ready lists if required */\r
- if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiRequestApcInterrupt(IN BOOLEAN NeedApc,\r
- IN UCHAR Processor)\r
-{\r
- /* Check if we need to request APC delivery */\r
- if (NeedApc)\r
- {\r
- /* Check if it's on another CPU */\r
- if (KeGetPcr()->Number != Cpu)\r
- {\r
- /* Send an IPI to request delivery */\r
- KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);\r
- }\r
- else\r
- {\r
- /* Request a software interrupt */\r
- HalRequestSoftwareInterrupt(APC_LEVEL);\r
- }\r
- }\r
-}\r
-\r
-#endif\r
-\r
-FORCEINLINE\r
-VOID\r
-KiAcquireApcLock(IN PKTHREAD Thread,\r
- IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Acquire the lock and raise to synchronization level */\r
- KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread,\r
- IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Acquire the lock */\r
- KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Release the lock */\r
- KeReleaseInStackQueuedSpinLock(Handle);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Release the lock */\r
- KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiAcquireProcessLock(IN PKPROCESS Process,\r
- IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Acquire the lock and raise to synchronization level */\r
- KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);\r
-}\r
-\r
-FORCEINLINE\r
-VOID\r
-KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)\r
-{\r
- /* Release the lock */\r
- KeReleaseInStackQueuedSpinLock(Handle);\r
-}\r
-\r
-//\r
-// This routine queues a thread that is ready on the PRCB's ready lists.\r
-// If this thread cannot currently run on this CPU, then the thread is\r
-// added to the deferred ready list instead.\r
-//\r
-// This routine must be entered with the PRCB lock held and it will exit\r
-// with the PRCB lock released!\r
-//\r
-FORCEINLINE\r
-VOID\r
-KxQueueReadyThread(IN PKTHREAD Thread,\r
- IN PKPRCB Prcb)\r
-{\r
- BOOLEAN Preempted;\r
- KPRIORITY Priority;\r
-\r
- /* Sanity checks */\r
- ASSERT(Prcb == KeGetCurrentPrcb());\r
- ASSERT(Thread->State == Running);\r
- ASSERT(Thread->NextProcessor == Prcb->Number);\r
-\r
- /* Check if this thread is allowed to run in this CPU */\r
-#ifdef _CONFIG_SMP\r
- if ((Thread->Affinity) & (Prcb->SetMember))\r
-#else\r
- if (TRUE)\r
-#endif\r
- {\r
- /* Set thread ready for execution */\r
- Thread->State = Ready;\r
-\r
- /* Save current priority and if someone had pre-empted it */\r
- Priority = Thread->Priority;\r
- Preempted = Thread->Preempted;\r
-\r
- /* We're not pre-empting now, and set the wait time */\r
- Thread->Preempted = FALSE;\r
- Thread->WaitTime = KeTickCount.LowPart;\r
-\r
- /* Sanity check */\r
- ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));\r
-\r
- /* Insert this thread in the appropriate order */\r
- Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],\r
- &Thread->WaitListEntry) :\r
- InsertTailList(&Prcb->DispatcherReadyListHead[Priority],\r
- &Thread->WaitListEntry);\r
-\r
- /* Update the ready summary */\r
- Prcb->ReadySummary |= PRIORITY_MASK(Priority);\r
-\r
- /* Sanity check */\r
- ASSERT(Priority == Thread->Priority);\r
-\r
- /* Release the PRCB lock */\r
- KiReleasePrcbLock(Prcb);\r
- }\r
- else\r
- {\r
- /* Otherwise, prepare this thread to be deferred */\r
- Thread->State = DeferredReady;\r
- Thread->DeferredProcessor = Prcb->Number;\r
-\r
- /* Release the lock and defer scheduling */\r
- KiReleasePrcbLock(Prcb);\r
- KiDeferredReadyThread(Thread);\r
- }\r
-}\r
-\r
-//\r
-// This routine scans for an appropriate ready thread to select at the\r
-// given priority and for the given CPU.\r
-//\r
-FORCEINLINE\r
-PKTHREAD\r
-KiSelectReadyThread(IN KPRIORITY Priority,\r
- IN PKPRCB Prcb)\r
-{\r
- LONG PriorityMask, PrioritySet, HighPriority;\r
- PLIST_ENTRY ListEntry;\r
- PKTHREAD Thread;\r
-\r
- /* Save the current mask and get the priority set for the CPU */\r
- PriorityMask = Priority;\r
- PrioritySet = Prcb->ReadySummary >> (UCHAR)Priority;\r
- if (!PrioritySet) return NULL;\r
-\r
- /* Get the highest priority possible */\r
- BitScanReverse((PULONG)&HighPriority, PrioritySet);\r
- ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);\r
- HighPriority += PriorityMask;\r
-\r
- /* Make sure the list isn't at highest priority */\r
- ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);\r
-\r
- /* Get the first thread on the list */\r
- ListEntry = &Prcb->DispatcherReadyListHead[HighPriority];\r
- Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);\r
-\r
- /* Make sure this thread is here for a reason */\r
- ASSERT(HighPriority == Thread->Priority);\r
- ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));\r
- ASSERT(Thread->NextProcessor == Prcb->Number);\r
-\r
- /* Remove it from the list */\r
- RemoveEntryList(&Thread->WaitListEntry);\r
- if (IsListEmpty(&Thread->WaitListEntry))\r
- {\r
- /* The list is empty now, reset the ready summary */\r
- Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);\r
- }\r
-\r
- /* Sanity check and return the thread */\r
- ASSERT((Thread == NULL) ||\r
- (Thread->BasePriority == 0) ||\r
- (Thread->Priority != 0));\r
- return Thread;\r
-}\r
-\r
-//\r
-// This routine computes the new priority for a thread. It is only valid for\r
-// threads with priorities in the dynamic priority range.\r
-//\r
-SCHAR\r
-FORCEINLINE\r
-KiComputeNewPriority(IN PKTHREAD Thread)\r
-{\r
- SCHAR Priority;\r
-\r
- /* Priority sanity checks */\r
- ASSERT((Thread->PriorityDecrement >= 0) &&\r
- (Thread->PriorityDecrement <= Thread->Priority));\r
- ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?\r
- TRUE : (Thread->PriorityDecrement == 0));\r
-\r
- /* Get the current priority */\r
- Priority = Thread->Priority;\r
- if (Priority < LOW_REALTIME_PRIORITY)\r
- {\r
- /* Set the New Priority and add the Priority Decrement */\r
- Priority += (Priority - Thread->PriorityDecrement - 1);\r
-\r
- /* Don't go out of bounds */\r
- if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;\r
-\r
- /* Reset the priority decrement */\r
- Thread->PriorityDecrement = 0;\r
- }\r
-\r
- /* Sanity check */\r
- ASSERT((Thread->BasePriority == 0) || (Priority != 0));\r
-\r
- /* Return the new priority */\r
- return Priority;\r
-}\r
-\r
+/*
+* PROJECT: ReactOS Kernel
+* LICENSE: GPL - See COPYING in the top level directory
+* FILE: ntoskrnl/include/ke_x.h
+* PURPOSE: Internal Inlined Functions for the Kernel
+* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
+*/
+
+//
+// Thread Dispatcher Header DebugActive Mask
+//
+#define DR_MASK(x) 1 << x
+#define DR_ACTIVE_MASK 0x10
+#define DR_REG_MASK 0x4F
+
+//
+// Sanitizes a selector
+//
+FORCEINLINE
+ULONG
+Ke386SanitizeSeg(IN ULONG Cs,
+ IN KPROCESSOR_MODE Mode)
+{
+ //
+ // Check if we're in kernel-mode, and force CPL 0 if so.
+ // Otherwise, force CPL 3.
+ //
+ return ((Mode == KernelMode) ?
+ (Cs & (0xFFFF & ~RPL_MASK)) :
+ (RPL_MASK | (Cs & 0xFFFF)));
+}
+
+//
+// Sanitizes EFLAGS
+//
+FORCEINLINE
+ULONG
+Ke386SanitizeFlags(IN ULONG Eflags,
+ IN KPROCESSOR_MODE Mode)
+{
+ //
+ // Check if we're in kernel-mode, and sanitize EFLAGS if so.
+ // Otherwise, also force interrupt mask on.
+ //
+ return ((Mode == KernelMode) ?
+ (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
+ (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
+}
+
+//
+// Gets a DR register from a CONTEXT structure
+//
+FORCEINLINE
+PVOID
+KiDrFromContext(IN ULONG Dr,
+ IN PCONTEXT Context)
+{
+ return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]);
+}
+
+//
+// Gets a DR register from a KTRAP_FRAME structure
+//
+FORCEINLINE
+PVOID*
+KiDrFromTrapFrame(IN ULONG Dr,
+ IN PKTRAP_FRAME TrapFrame)
+{
+ return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]);
+}
+
+//
+//
+//
+FORCEINLINE
+PVOID
+Ke386SanitizeDr(IN PVOID DrAddress,
+ IN KPROCESSOR_MODE Mode)
+{
+ //
+ // Check if we're in kernel-mode, and return the address directly if so.
+ // Otherwise, make sure it's not inside the kernel-mode address space.
+ // If it is, then clear the address.
+ //
+ return ((Mode == KernelMode) ? DrAddress :
+ (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
+}
+
+//
+// Enters a Guarded Region
+//
+#define KeEnterGuardedRegion() \
+{ \
+ PKTHREAD _Thread = KeGetCurrentThread(); \
+ \
+ /* Sanity checks */ \
+ ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
+ ASSERT(_Thread == KeGetCurrentThread()); \
+ ASSERT((_Thread->SpecialApcDisable <= 0) && \
+ (_Thread->SpecialApcDisable != -32768)); \
+ \
+ /* Disable Special APCs */ \
+ _Thread->SpecialApcDisable--; \
+}
+
+//
+// Leaves a Guarded Region
+//
+#define KeLeaveGuardedRegion() \
+{ \
+ PKTHREAD _Thread = KeGetCurrentThread(); \
+ \
+ /* Sanity checks */ \
+ ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); \
+ ASSERT(_Thread == KeGetCurrentThread()); \
+ ASSERT(_Thread->SpecialApcDisable < 0); \
+ \
+ /* Leave region and check if APCs are OK now */ \
+ if (!(++_Thread->SpecialApcDisable)) \
+ { \
+ /* Check for Kernel APCs on the list */ \
+ if (!IsListEmpty(&_Thread->ApcState. \
+ ApcListHead[KernelMode])) \
+ { \
+ /* Check for APC Delivery */ \
+ KiCheckForKernelApcDelivery(); \
+ } \
+ } \
+}
+
+//
+// TODO: Guarded Mutex Routines
+//
+
+//
+// Enters a Critical Region
+//
+#define KeEnterCriticalRegion() \
+{ \
+ PKTHREAD _Thread = KeGetCurrentThread(); \
+ \
+ /* Sanity checks */ \
+ ASSERT(_Thread == KeGetCurrentThread()); \
+ ASSERT((_Thread->KernelApcDisable <= 0) && \
+ (_Thread->KernelApcDisable != -32768)); \
+ \
+ /* Disable Kernel APCs */ \
+ _Thread->KernelApcDisable--; \
+}
+
+//
+// Leaves a Critical Region
+//
+#define KeLeaveCriticalRegion() \
+{ \
+ PKTHREAD _Thread = KeGetCurrentThread(); \
+ \
+ /* Sanity checks */ \
+ ASSERT(_Thread == KeGetCurrentThread()); \
+ ASSERT(_Thread->KernelApcDisable < 0); \
+ \
+ /* Enable Kernel APCs */ \
+ _Thread->KernelApcDisable++; \
+ \
+ /* Check if Kernel APCs are now enabled */ \
+ if (!(_Thread->KernelApcDisable)) \
+ { \
+ /* Check if we need to request an APC Delivery */ \
+ if (!(IsListEmpty(&_Thread->ApcState.ApcListHead[KernelMode])) && \
+ !(_Thread->SpecialApcDisable)) \
+ { \
+ /* Check for the right environment */ \
+ KiCheckForKernelApcDelivery(); \
+ } \
+ } \
+}
+
+//
+// Satisfies the wait of any dispatcher object
+//
+#define KiSatisfyObjectWait(Object, Thread) \
+{ \
+ /* Special case for Mutants */ \
+ if ((Object)->Header.Type == MutantObject) \
+ { \
+ /* Decrease the Signal State */ \
+ (Object)->Header.SignalState--; \
+ \
+ /* Check if it's now non-signaled */ \
+ if (!(Object)->Header.SignalState) \
+ { \
+ /* Set the Owner Thread */ \
+ (Object)->OwnerThread = Thread; \
+ \
+ /* Disable APCs if needed */ \
+ Thread->KernelApcDisable -= (Object)->ApcDisable; \
+ \
+ /* Check if it's abandoned */ \
+ if ((Object)->Abandoned) \
+ { \
+ /* Unabandon it */ \
+ (Object)->Abandoned = FALSE; \
+ \
+ /* Return Status */ \
+ Thread->WaitStatus = STATUS_ABANDONED; \
+ } \
+ \
+ /* Insert it into the Mutant List */ \
+ InsertHeadList(Thread->MutantListHead.Blink, \
+ &(Object)->MutantListEntry); \
+ } \
+ } \
+ else if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
+ EventSynchronizationObject) \
+ { \
+ /* Synchronization Timers and Events just get un-signaled */ \
+ (Object)->Header.SignalState = 0; \
+ } \
+ else if ((Object)->Header.Type == SemaphoreObject) \
+ { \
+ /* These ones can have multiple states, so we only decrease it */ \
+ (Object)->Header.SignalState--; \
+ } \
+}
+
+//
+// Satisfies the wait of a mutant dispatcher object
+//
+#define KiSatisfyMutantWait(Object, Thread) \
+{ \
+ /* Decrease the Signal State */ \
+ (Object)->Header.SignalState--; \
+ \
+ /* Check if it's now non-signaled */ \
+ if (!(Object)->Header.SignalState) \
+ { \
+ /* Set the Owner Thread */ \
+ (Object)->OwnerThread = Thread; \
+ \
+ /* Disable APCs if needed */ \
+ Thread->KernelApcDisable -= (Object)->ApcDisable; \
+ \
+ /* Check if it's abandoned */ \
+ if ((Object)->Abandoned) \
+ { \
+ /* Unabandon it */ \
+ (Object)->Abandoned = FALSE; \
+ \
+ /* Return Status */ \
+ Thread->WaitStatus = STATUS_ABANDONED; \
+ } \
+ \
+ /* Insert it into the Mutant List */ \
+ InsertHeadList(Thread->MutantListHead.Blink, \
+ &(Object)->MutantListEntry); \
+ } \
+}
+
+//
+// Satisfies the wait of any nonmutant dispatcher object
+//
+#define KiSatisfyNonMutantWait(Object, Thread) \
+{ \
+ if (((Object)->Header.Type & TIMER_OR_EVENT_TYPE) == \
+ EventSynchronizationObject) \
+ { \
+ /* Synchronization Timers and Events just get un-signaled */ \
+ (Object)->Header.SignalState = 0; \
+ } \
+ else if ((Object)->Header.Type == SemaphoreObject) \
+ { \
+ /* These ones can have multiple states, so we only decrease it */ \
+ (Object)->Header.SignalState--; \
+ } \
+}
+
+//
+// Recalculates the due time
+//
+PLARGE_INTEGER
+FORCEINLINE
+KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
+ IN PLARGE_INTEGER DueTime,
+ IN OUT PLARGE_INTEGER NewDueTime)
+{
+ /* Don't do anything for absolute waits */
+ if (OriginalDueTime->QuadPart >= 0) return OriginalDueTime;
+
+ /* Otherwise, query the interrupt time and recalculate */
+ NewDueTime->QuadPart = KeQueryInterruptTime();
+ NewDueTime->QuadPart -= DueTime->QuadPart;
+ return NewDueTime;
+}
+
+//
+// Determines wether a thread should be added to the wait list
+//
+FORCEINLINE
+BOOLEAN
+KiCheckThreadStackSwap(IN PKTHREAD Thread,
+ IN KPROCESSOR_MODE WaitMode)
+{
+ /* Check the required conditions */
+ if ((WaitMode != KernelMode) &&
+ (Thread->EnableStackSwap) &&
+ (Thread->Priority >= (LOW_REALTIME_PRIORITY + 9)))
+ {
+ /* We are go for swap */
+ return TRUE;
+ }
+ else
+ {
+ /* Don't swap the thread */
+ return FALSE;
+ }
+}
+
+//
+// Adds a thread to the wait list
+//
+#define KiAddThreadToWaitList(Thread, Swappable) \
+{ \
+ /* Make sure it's swappable */ \
+ if (Swappable) \
+ { \
+ /* Insert it into the PRCB's List */ \
+ InsertTailList(&KeGetCurrentPrcb()->WaitListHead, \
+ &Thread->WaitListEntry); \
+ } \
+}
+
+//
+// Checks if a wait in progress should be interrupted by APCs or an alertable
+// state.
+//
+FORCEINLINE
+NTSTATUS
+KiCheckAlertability(IN PKTHREAD Thread,
+ IN BOOLEAN Alertable,
+ IN KPROCESSOR_MODE WaitMode)
+{
+ /* Check if the wait is alertable */
+ if (Alertable)
+ {
+ /* It is, first check if the thread is alerted in this mode */
+ if (Thread->Alerted[WaitMode])
+ {
+ /* It is, so bail out of the wait */
+ Thread->Alerted[WaitMode] = FALSE;
+ return STATUS_ALERTED;
+ }
+ else if ((WaitMode != KernelMode) &&
+ (!IsListEmpty(&Thread->ApcState.ApcListHead[UserMode])))
+ {
+ /* It's isn't, but this is a user wait with queued user APCs */
+ Thread->ApcState.UserApcPending = TRUE;
+ return STATUS_USER_APC;
+ }
+ else if (Thread->Alerted[KernelMode])
+ {
+ /* It isn't that either, but we're alered in kernel mode */
+ Thread->Alerted[KernelMode] = FALSE;
+ return STATUS_ALERTED;
+ }
+ }
+ else if ((WaitMode != KernelMode) && (Thread->ApcState.UserApcPending))
+ {
+ /* Not alertable, but this is a user wait with pending user APCs */
+ return STATUS_USER_APC;
+ }
+
+ /* Otherwise, we're fine */
+ return STATUS_WAIT_0;
+}
+
+FORCEINLINE
+BOOLEAN
+KxDelayThreadWait(IN PKTHREAD Thread,
+ IN BOOLEAN Alertable,
+ IN KPROCESSOR_MODE WaitMode)
+{
+ BOOLEAN Swappable;
+ PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+
+ /* Setup the Wait Block */
+ Thread->WaitBlockList = TimerBlock;
+ TimerBlock->NextWaitBlock = TimerBlock;
+
+ /* Link the timer to this Wait Block */
+ Thread->Timer.Header.WaitListHead.Flink = &TimerBlock->WaitListEntry;
+ Thread->Timer.Header.WaitListHead.Blink = &TimerBlock->WaitListEntry;
+
+ /* Clear wait status */
+ Thread->WaitStatus = STATUS_WAIT_0;
+
+ /* Setup wait fields */
+ Thread->Alertable = Alertable;
+ Thread->WaitReason = DelayExecution;
+ Thread->WaitMode = WaitMode;
+
+ /* Check if we can swap the thread's stack */
+ Thread->WaitListEntry.Flink = NULL;
+ Swappable = KiCheckThreadStackSwap(Thread, WaitMode);
+
+ /* Set the wait time */
+ Thread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
+ return Swappable;
+}
+
+FORCEINLINE
+BOOLEAN
+KxMultiThreadWait(IN PKTHREAD Thread,
+ IN PKWAIT_BLOCK WaitBlock,
+ IN BOOLEAN Alertable,
+ IN KWAIT_REASON WaitReason,
+ IN KPROCESSOR_MODE WaitMode)
+{
+ BOOLEAN Swappable;
+ PKTIMER ThreadTimer = &Thread->Timer;
+
+ /* Set default wait status */
+ Thread->WaitStatus = STATUS_WAIT_0;
+
+ /* Link wait block array to the thread */
+ Thread->WaitBlockList = WaitBlock;
+
+ /* Initialize the timer list */
+ InitializeListHead(&ThreadTimer->Header.WaitListHead);
+
+ /* Set wait settings */
+ Thread->Alertable = Alertable;
+ Thread->WaitMode = WaitMode;
+ Thread->WaitReason = WaitReason;
+
+ /* Check if we can swap the thread's stack */
+ Thread->WaitListEntry.Flink = NULL;
+ Swappable = KiCheckThreadStackSwap(Thread, WaitMode);
+
+ /* Set the wait time */
+ Thread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
+ return Swappable;
+}
+
+FORCEINLINE
+BOOLEAN
+KxSingleThreadWait(IN PKTHREAD Thread,
+ IN PKWAIT_BLOCK WaitBlock,
+ IN PVOID Object,
+ IN PLARGE_INTEGER Timeout,
+ IN BOOLEAN Alertable,
+ IN KWAIT_REASON WaitReason,
+ IN KPROCESSOR_MODE WaitMode)
+{
+ BOOLEAN Swappable;
+ PKWAIT_BLOCK TimerBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
+
+ /* Setup the Wait Block */
+ Thread->WaitBlockList = WaitBlock;
+ WaitBlock->WaitKey = STATUS_WAIT_0;
+ WaitBlock->Object = Object;
+ WaitBlock->WaitType = WaitAny;
+
+ /* Clear wait status */
+ Thread->WaitStatus = STATUS_WAIT_0;
+
+ /* Check if we have a timer */
+ if (Timeout)
+ {
+ /* Pointer to timer block */
+ WaitBlock->NextWaitBlock = TimerBlock;
+ TimerBlock->NextWaitBlock = WaitBlock;
+
+ /* Link the timer to this Wait Block */
+ Thread->Timer.Header.WaitListHead.Flink = &TimerBlock->WaitListEntry;
+ Thread->Timer.Header.WaitListHead.Blink = &TimerBlock->WaitListEntry;
+ }
+ else
+ {
+ /* No timer block, just ourselves */
+ WaitBlock->NextWaitBlock = WaitBlock;
+ }
+
+ /* Setup wait fields */
+ Thread->Alertable = Alertable;
+ Thread->WaitReason = WaitReason;
+ Thread->WaitMode = WaitMode;
+
+ /* Check if we can swap the thread's stack */
+ Thread->WaitListEntry.Flink = NULL;
+ Swappable = KiCheckThreadStackSwap(Thread, WaitMode);
+
+ /* Set the wait time */
+ Thread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
+ return Swappable;
+}
+
+//
+// Unwaits a Thread
+//
+FORCEINLINE
+VOID
+KxUnwaitThread(IN DISPATCHER_HEADER *Object,
+ IN KPRIORITY Increment)
+{
+ PLIST_ENTRY WaitEntry, WaitList;
+ PKWAIT_BLOCK WaitBlock;
+ PKTHREAD WaitThread;
+ ULONG WaitKey;
+
+ /* Loop the Wait Entries */
+ WaitList = &Object->WaitListHead;
+ ASSERT(IsListEmpty(&Object->WaitListHead) == FALSE);
+ WaitEntry = WaitList->Flink;
+ do
+ {
+ /* Get the current wait block */
+ WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
+
+ /* Get the waiting thread */
+ WaitThread = WaitBlock->Thread;
+
+ /* Check the current Wait Mode */
+ if (WaitBlock->WaitType == WaitAny)
+ {
+ /* Use the actual wait key */
+ WaitKey = WaitBlock->WaitKey;
+ }
+ else
+ {
+ /* Otherwise, use STATUS_KERNEL_APC */
+ WaitKey = STATUS_KERNEL_APC;
+ }
+
+ /* Unwait the thread */
+ KiUnwaitThread(WaitThread, WaitKey, Increment);
+
+ /* Next entry */
+ WaitEntry = WaitList->Flink;
+ } while (WaitEntry != WaitList);
+}
+
+//
+// Unwaits a Thread waiting on an event
+//
+FORCEINLINE
+VOID
+KxUnwaitThreadForEvent(IN PKEVENT Event,
+ IN KPRIORITY Increment)
+{
+ PLIST_ENTRY WaitEntry, WaitList;
+ PKWAIT_BLOCK WaitBlock;
+ PKTHREAD WaitThread;
+
+ /* Loop the Wait Entries */
+ WaitList = &Event->Header.WaitListHead;
+ ASSERT(IsListEmpty(&Event->Header.WaitListHead) == FALSE);
+ WaitEntry = WaitList->Flink;
+ do
+ {
+ /* Get the current wait block */
+ WaitBlock = CONTAINING_RECORD(WaitEntry, KWAIT_BLOCK, WaitListEntry);
+
+ /* Get the waiting thread */
+ WaitThread = WaitBlock->Thread;
+
+ /* Check the current Wait Mode */
+ if (WaitBlock->WaitType == WaitAny)
+ {
+ /* Un-signal it */
+ Event->Header.SignalState = 0;
+
+ /* Un-signal the event and unwait the thread */
+ KiUnwaitThread(WaitThread, WaitBlock->WaitKey, Increment);
+ break;
+ }
+
+ /* Unwait the thread with STATUS_KERNEL_APC */
+ KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
+
+ /* Next entry */
+ WaitEntry = WaitList->Flink;
+ } while (WaitEntry != WaitList);
+}
+
+#ifndef _CONFIG_SMP
+//
+// Spinlock Acquire at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+ /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
+ UNREFERENCED_PARAMETER(SpinLock);
+}
+
+//
+// Spinlock Release at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+ /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
+ UNREFERENCED_PARAMETER(SpinLock);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+VOID
+FORCEINLINE
+KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
+{
+ UNREFERENCED_PARAMETER(Object);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+VOID
+FORCEINLINE
+KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
+{
+ UNREFERENCED_PARAMETER(Object);
+}
+
+KIRQL
+FORCEINLINE
+KiAcquireDispatcherLock(VOID)
+{
+ /* Raise to DPC level */
+ return KeRaiseIrqlToDpcLevel();
+}
+
+VOID
+FORCEINLINE
+KiReleaseDispatcherLock(IN KIRQL OldIrql)
+{
+ /* Just exit the dispatcher */
+ KiExitDispatcher(OldIrql);
+}
+
+VOID
+FORCEINLINE
+KiAcquireDispatcherLockAtDpcLevel(VOID)
+{
+ /* This is a no-op at DPC Level for UP systems */
+ return;
+}
+
+VOID
+FORCEINLINE
+KiReleaseDispatcherLockFromDpcLevel(VOID)
+{
+ /* This is a no-op at DPC Level for UP systems */
+ return;
+}
+
+//
+// This routine makes the thread deferred ready on the boot CPU.
+//
+FORCEINLINE
+VOID
+KiInsertDeferredReadyList(IN PKTHREAD Thread)
+{
+ /* Set the thread to deferred state and boot CPU */
+ Thread->State = DeferredReady;
+ Thread->DeferredProcessor = 0;
+
+ /* Make the thread ready immediately */
+ KiDeferredReadyThread(Thread);
+}
+
+FORCEINLINE
+VOID
+KiRescheduleThread(IN BOOLEAN NewThread,
+ IN ULONG Cpu)
+{
+ /* This is meaningless on UP systems */
+ UNREFERENCED_PARAMETER(NewThread);
+ UNREFERENCED_PARAMETER(Cpu);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+VOID
+KiSetThreadSwapBusy(IN PKTHREAD Thread)
+{
+ UNREFERENCED_PARAMETER(Thread);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+VOID
+KiAcquirePrcbLock(IN PKPRCB Prcb)
+{
+ UNREFERENCED_PARAMETER(Prcb);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+VOID
+KiReleasePrcbLock(IN PKPRCB Prcb)
+{
+ UNREFERENCED_PARAMETER(Prcb);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+VOID
+KiAcquireThreadLock(IN PKTHREAD Thread)
+{
+ UNREFERENCED_PARAMETER(Thread);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+VOID
+KiReleaseThreadLock(IN PKTHREAD Thread)
+{
+ UNREFERENCED_PARAMETER(Thread);
+}
+
+//
+// This routine protects against multiple CPU acquires, it's meaningless on UP.
+//
+FORCEINLINE
+BOOLEAN
+KiTryThreadLock(IN PKTHREAD Thread)
+{
+ UNREFERENCED_PARAMETER(Thread);
+ return FALSE;
+}
+
+FORCEINLINE
+VOID
+KiCheckDeferredReadyList(IN PKPRCB Prcb)
+{
+ /* There are no deferred ready lists on UP systems */
+ UNREFERENCED_PARAMETER(Prcb);
+}
+
+FORCEINLINE
+VOID
+KiRundownThread(IN PKTHREAD Thread)
+{
+ /* Check if this is the NPX Thread */
+ if (KeGetCurrentPrcb()->NpxThread == Thread)
+ {
+ /* Clear it */
+ KeGetCurrentPrcb()->NpxThread = NULL;
+ Ke386FnInit();
+ }
+}
+
+FORCEINLINE
+VOID
+KiRequestApcInterrupt(IN BOOLEAN NeedApc,
+ IN UCHAR Processor)
+{
+ /* We deliver instantly on UP */
+ UNREFERENCED_PARAMETER(NeedApc);
+ UNREFERENCED_PARAMETER(Processor);
+}
+
+#else
+
+//
+// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+ for (;;)
+ {
+ /* Try to acquire it */
+ if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
+ {
+ /* Value changed... wait until it's locked */
+ while (*(volatile KSPIN_LOCK *)SpinLock == 1)
+ {
+#ifdef DBG
+ /* On debug builds, we use a much slower but useful routine */
+ Kii386SpinOnSpinLock(SpinLock, 5);
+#else
+ /* Otherwise, just yield and keep looping */
+ YieldProcessor();
+#endif
+ }
+ }
+ else
+ {
+#ifdef DBG
+ /* On debug builds, we OR in the KTHREAD */
+ *SpinLock = KeGetCurrentThread() | 1;
+#endif
+ /* All is well, break out */
+ break;
+ }
+ }
+}
+
+//
+// Spinlock Release at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+#ifdef DBG
+ /* Make sure that the threads match */
+ if ((KeGetCurrentThread() | 1) != *SpinLock)
+ {
+ /* They don't, bugcheck */
+ KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0);
+ }
+#endif
+ /* Clear the lock */
+ InterlockedAnd(SpinLock, 0);
+}
+
+KIRQL
+FORCEINLINE
+KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
+{
+ LONG OldValue, NewValue;
+
+ /* Make sure we're at a safe level to touch the lock */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
+
+ /* Start acquire loop */
+ do
+ {
+ /* Loop until the other CPU releases it */
+ while ((UCHAR)Object->Lock & KOBJECT_LOCK_BIT)
+ {
+ /* Let the CPU know that this is a loop */
+ YieldProcessor();
+ };
+
+ /* Try acquiring the lock now */
+ NewValue = InterlockedCompareExchange(&Object->Lock,
+ OldValue | KOBJECT_LOCK_BIT,
+ OldValue);
+ } while (NewValue != OldValue);
+}
+
+KIRQL
+FORCEINLINE
+KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object)
+{
+ /* Make sure we're at a safe level to touch the lock */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
+
+ /* Release it */
+ InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT);
+}
+
+KIRQL
+FORCEINLINE
+KiAcquireDispatcherLock(VOID)
+{
+ /* Raise to synchronization level and acquire the dispatcher lock */
+ return KeAcquireQueuedSpinLockRaiseToSynch(LockQueueDispatcherLock);
+}
+
+VOID
+FORCEINLINE
+KiReleaseDispatcherLock(IN KIRQL OldIrql)
+{
+ /* First release the lock */
+ KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
+ LockQueue[LockQueueDispatcherLock]);
+
+ /* Then exit the dispatcher */
+ KiExitDispatcher(OldIrql);
+}
+
+//
+// This routine inserts a thread into the deferred ready list of the given CPU
+//
+FORCEINLINE
+VOID
+KiInsertDeferredReadyList(IN PKTHREAD Thread)
+{
+ PKPRCB Prcb = KeGetCurrentPrcb();
+
+ /* Set the thread to deferred state and CPU */
+ Thread->State = DeferredReady;
+ Thread->DeferredProcessor = Prcb->Number;
+
+ /* Add it on the list */
+ PushEntryList(&Prcb->DeferredReadyListHead, &Thread->SwapListEntry);
+}
+
+FORCEINLINE
+VOID
+KiRescheduleThread(IN BOOLEAN NewThread,
+ IN ULONG Cpu)
+{
+ /* Check if a new thread needs to be scheduled on a different CPU */
+ if ((NewThread) && !(KeGetPcr()->Number == Cpu))
+ {
+ /* Send an IPI to request delivery */
+ KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);
+ }
+}
+
+//
+// This routine sets the current thread in a swap busy state, which ensure that
+// nobody else tries to swap it concurrently.
+//
+FORCEINLINE
+VOID
+KiSetThreadSwapBusy(IN PKTHREAD Thread)
+{
+ /* Make sure nobody already set it */
+ ASSERT(Thread->SwapBusy == FALSE);
+
+ /* Set it ourselves */
+ Thread->SwapBusy = TRUE;
+}
+
+//
+// This routine acquires the PRCB lock so that only one caller can touch
+// volatile PRCB data.
+//
+// Since this is a simple optimized spin-lock, it must be be only acquired
+// at dispatcher level or higher!
+//
+FORCEINLINE
+VOID
+KiAcquirePrcbLock(IN PKPRCB Prcb)
+{
+ /* Make sure we're at a safe level to touch the PRCB lock */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
+
+ /* Start acquire loop */
+ for (;;)
+ {
+ /* Acquire the lock and break out if we acquired it first */
+ if (!InterlockedExchange(&Prcb->PrcbLock, 1)) break;
+
+ /* Loop until the other CPU releases it */
+ do
+ {
+ /* Let the CPU know that this is a loop */
+ YieldProcessor();
+ } while (Prcb->PrcbLock);
+ }
+}
+
+//
+// This routine releases the PRCB lock so that other callers can touch
+// volatile PRCB data.
+//
+// Since this is a simple optimized spin-lock, it must be be only acquired
+// at dispatcher level or higher!
+//
+FORCEINLINE
+VOID
+KiReleasePrcbLock(IN PKPRCB Prcb)
+{
+ /* Make sure it's acquired! */
+ ASSERT(Prcb->PrcbLock != 0);
+
+ /* Release it */
+ InterlockedAnd(&Prcb->PrcbLock, 0);
+}
+
+//
+// This routine acquires the thread lock so that only one caller can touch
+// volatile thread data.
+//
+// Since this is a simple optimized spin-lock, it must be be only acquired
+// at dispatcher level or higher!
+//
+FORCEINLINE
+VOID
+KiAcquireThreadLock(IN PKTHREAD Thread)
+{
+ /* Make sure we're at a safe level to touch the thread lock */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
+
+ /* Start acquire loop */
+ for (;;)
+ {
+ /* Acquire the lock and break out if we acquired it first */
+ if (!InterlockedExchange(&Thread->ThreadLock, 1)) break;
+
+ /* Loop until the other CPU releases it */
+ do
+ {
+ /* Let the CPU know that this is a loop */
+ YieldProcessor();
+ } while (Thread->ThreadLock);
+ }
+}
+
+//
+// This routine releases the thread lock so that other callers can touch
+// volatile thread data.
+//
+// Since this is a simple optimized spin-lock, it must be be only acquired
+// at dispatcher level or higher!
+//
+FORCEINLINE
+VOID
+KiReleaseThreadLock(IN PKTHREAD Thread)
+{
+ /* Release it */
+ InterlockedAnd(&Thread->ThreadLock, 0);
+}
+
+FORCEINLINE
+BOOLEAN
+KiTryThreadLock(IN PKTHREAD Thread)
+{
+ LONG Value;
+
+ /* If the lock isn't acquired, return false */
+ if (!Thread->ThreadLock) return FALSE;
+
+ /* Otherwise, try to acquire it and check the result */
+ Value = 1;
+ Value = InterlockedExchange(&Thread->ThreadLock, &Value);
+
+ /* Return the lock state */
+ return (Value == TRUE);
+}
+
+FORCEINLINE
+VOID
+KiCheckDeferredReadyList(IN PKPRCB Prcb)
+{
+ /* Scan the deferred ready lists if required */
+ if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
+}
+
+FORCEINLINE
+VOID
+KiRequestApcInterrupt(IN BOOLEAN NeedApc,
+ IN UCHAR Processor)
+{
+ /* Check if we need to request APC delivery */
+ if (NeedApc)
+ {
+ /* Check if it's on another CPU */
+ if (KeGetPcr()->Number != Cpu)
+ {
+ /* Send an IPI to request delivery */
+ KiIpiSendRequest(AFFINITY_MASK(Cpu), IPI_DPC);
+ }
+ else
+ {
+ /* Request a software interrupt */
+ HalRequestSoftwareInterrupt(APC_LEVEL);
+ }
+ }
+}
+
+#endif
+
+FORCEINLINE
+VOID
+KiAcquireApcLock(IN PKTHREAD Thread,
+ IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Acquire the lock and raise to synchronization level */
+ KeAcquireInStackQueuedSpinLockRaiseToSynch(&Thread->ApcQueueLock, Handle);
+}
+
+FORCEINLINE
+VOID
+KiAcquireApcLockAtDpcLevel(IN PKTHREAD Thread,
+ IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Acquire the lock */
+ KeAcquireInStackQueuedSpinLockAtDpcLevel(&Thread->ApcQueueLock, Handle);
+}
+
+FORCEINLINE
+VOID
+KiAcquireApcLockAtApcLevel(IN PKTHREAD Thread,
+ IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Acquire the lock */
+ KeAcquireInStackQueuedSpinLock(&Thread->ApcQueueLock, Handle);
+}
+
+FORCEINLINE
+VOID
+KiReleaseApcLock(IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Release the lock */
+ KeReleaseInStackQueuedSpinLock(Handle);
+}
+
+FORCEINLINE
+VOID
+KiReleaseApcLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Release the lock */
+ KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
+}
+
+FORCEINLINE
+VOID
+KiAcquireProcessLock(IN PKPROCESS Process,
+ IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Acquire the lock and raise to synchronization level */
+ KeAcquireInStackQueuedSpinLockRaiseToSynch(&Process->ProcessLock, Handle);
+}
+
+FORCEINLINE
+VOID
+KiReleaseProcessLock(IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Release the lock */
+ KeReleaseInStackQueuedSpinLock(Handle);
+}
+
+FORCEINLINE
+VOID
+KiReleaseProcessLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE Handle)
+{
+ /* Release the lock */
+ KeReleaseInStackQueuedSpinLockFromDpcLevel(Handle);
+}
+
+FORCEINLINE
+VOID
+KiAcquireDeviceQueueLock(IN PKDEVICE_QUEUE DeviceQueue,
+ IN PKLOCK_QUEUE_HANDLE DeviceLock)
+{
+ /* Check if we were called from a threaded DPC */
+ if (KeGetCurrentPrcb()->DpcThreadActive)
+ {
+ /* Lock the Queue, we're not at DPC level */
+ KeAcquireInStackQueuedSpinLock(&DeviceQueue->Lock, DeviceLock);
+ }
+ else
+ {
+ /* We must be at DPC level, acquire the lock safely */
+ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+ KeAcquireInStackQueuedSpinLockAtDpcLevel(&DeviceQueue->Lock,
+ DeviceLock);
+ }
+}
+
+FORCEINLINE
+VOID
+KiReleaseDeviceQueueLock(IN PKLOCK_QUEUE_HANDLE DeviceLock)
+{
+ /* Check if we were called from a threaded DPC */
+ if (KeGetCurrentPrcb()->DpcThreadActive)
+ {
+ /* Unlock the Queue, we're not at DPC level */
+ KeReleaseInStackQueuedSpinLock(DeviceLock);
+ }
+ else
+ {
+ /* We must be at DPC level, release the lock safely */
+ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+ KeReleaseInStackQueuedSpinLockFromDpcLevel(DeviceLock);
+ }
+}
+
+//
+// This routine queues a thread that is ready on the PRCB's ready lists.
+// If this thread cannot currently run on this CPU, then the thread is
+// added to the deferred ready list instead.
+//
+// This routine must be entered with the PRCB lock held and it will exit
+// with the PRCB lock released!
+//
+FORCEINLINE
+VOID
+KxQueueReadyThread(IN PKTHREAD Thread,
+ IN PKPRCB Prcb)
+{
+ BOOLEAN Preempted;
+ KPRIORITY Priority;
+
+ /* Sanity checks */
+ ASSERT(Prcb == KeGetCurrentPrcb());
+ ASSERT(Thread->State == Running);
+ ASSERT(Thread->NextProcessor == Prcb->Number);
+
+ /* Check if this thread is allowed to run in this CPU */
+#ifdef _CONFIG_SMP
+ if ((Thread->Affinity) & (Prcb->SetMember))
+#else
+ if (TRUE)
+#endif
+ {
+ /* Set thread ready for execution */
+ Thread->State = Ready;
+
+ /* Save current priority and if someone had pre-empted it */
+ Priority = Thread->Priority;
+ Preempted = Thread->Preempted;
+
+ /* We're not pre-empting now, and set the wait time */
+ Thread->Preempted = FALSE;
+ Thread->WaitTime = KeTickCount.LowPart;
+
+ /* Sanity check */
+ ASSERT((Priority >= 0) && (Priority <= HIGH_PRIORITY));
+
+ /* Insert this thread in the appropriate order */
+ Preempted ? InsertHeadList(&Prcb->DispatcherReadyListHead[Priority],
+ &Thread->WaitListEntry) :
+ InsertTailList(&Prcb->DispatcherReadyListHead[Priority],
+ &Thread->WaitListEntry);
+
+ /* Update the ready summary */
+ Prcb->ReadySummary |= PRIORITY_MASK(Priority);
+
+ /* Sanity check */
+ ASSERT(Priority == Thread->Priority);
+
+ /* Release the PRCB lock */
+ KiReleasePrcbLock(Prcb);
+ }
+ else
+ {
+ /* Otherwise, prepare this thread to be deferred */
+ Thread->State = DeferredReady;
+ Thread->DeferredProcessor = Prcb->Number;
+
+ /* Release the lock and defer scheduling */
+ KiReleasePrcbLock(Prcb);
+ KiDeferredReadyThread(Thread);
+ }
+}
+
+//
+// This routine scans for an appropriate ready thread to select at the
+// given priority and for the given CPU.
+//
+FORCEINLINE
+PKTHREAD
+KiSelectReadyThread(IN KPRIORITY Priority,
+ IN PKPRCB Prcb)
+{
+ LONG PriorityMask, PrioritySet, HighPriority;
+ PLIST_ENTRY ListEntry;
+ PKTHREAD Thread = NULL;
+
+ /* Save the current mask and get the priority set for the CPU */
+ PriorityMask = Priority;
+ PrioritySet = Prcb->ReadySummary >> (UCHAR)Priority;
+ if (!PrioritySet) goto Quickie;
+
+ /* Get the highest priority possible */
+ BitScanReverse((PULONG)&HighPriority, PrioritySet);
+ ASSERT((PrioritySet & PRIORITY_MASK(HighPriority)) != 0);
+ HighPriority += PriorityMask;
+
+ /* Make sure the list isn't at highest priority */
+ ASSERT(IsListEmpty(&Prcb->DispatcherReadyListHead[HighPriority]) == FALSE);
+
+ /* Get the first thread on the list */
+ ListEntry = Prcb->DispatcherReadyListHead[HighPriority].Flink;
+ Thread = CONTAINING_RECORD(ListEntry, KTHREAD, WaitListEntry);
+
+ /* Make sure this thread is here for a reason */
+ ASSERT(HighPriority == Thread->Priority);
+ ASSERT(Thread->Affinity & AFFINITY_MASK(Prcb->Number));
+ ASSERT(Thread->NextProcessor == Prcb->Number);
+
+ /* Remove it from the list */
+ if (RemoveEntryList(&Thread->WaitListEntry))
+ {
+ /* The list is empty now, reset the ready summary */
+ Prcb->ReadySummary ^= PRIORITY_MASK(HighPriority);
+ }
+
+ /* Sanity check and return the thread */
+Quickie:
+ ASSERT((Thread == NULL) ||
+ (Thread->BasePriority == 0) ||
+ (Thread->Priority != 0));
+ return Thread;
+}
+
+//
+// This routine computes the new priority for a thread. It is only valid for
+// threads with priorities in the dynamic priority range.
+//
+SCHAR
+FORCEINLINE
+KiComputeNewPriority(IN PKTHREAD Thread,
+ IN SCHAR Adjustment)
+{
+ SCHAR Priority;
+
+ /* Priority sanity checks */
+ ASSERT((Thread->PriorityDecrement >= 0) &&
+ (Thread->PriorityDecrement <= Thread->Priority));
+ ASSERT((Thread->Priority < LOW_REALTIME_PRIORITY) ?
+ TRUE : (Thread->PriorityDecrement == 0));
+
+ /* Get the current priority */
+ Priority = Thread->Priority;
+ if (Priority < LOW_REALTIME_PRIORITY)
+ {
+ /* Decrease priority by the priority decrement */
+ Priority -= (Thread->PriorityDecrement + Adjustment);
+
+ /* Don't go out of bounds */
+ if (Priority < Thread->BasePriority) Priority = Thread->BasePriority;
+
+ /* Reset the priority decrement */
+ Thread->PriorityDecrement = 0;
+ }
+
+ /* Sanity check */
+ ASSERT((Thread->BasePriority == 0) || (Priority != 0));
+
+ /* Return the new priority */
+ return Priority;
+}
+
+PRKTHREAD
+FORCEINLINE
+KeGetCurrentThread(VOID)
+{
+ /* Return the current thread */
+ return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
+}
+
+UCHAR
+FORCEINLINE
+KeGetPreviousMode(VOID)
+{
+ /* Return the current mode */
+ return KeGetCurrentThread()->PreviousMode;
+}
projects / reactos.git / blobdiff