* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
*/
-//
-// Thread Dispatcher Header DebugActive Mask
-//
-#define DR_MASK(x) 1 << x
-#define DR_ACTIVE_MASK 0x10
-#define DR_REG_MASK 0x4F
-
-#ifdef _M_IX86
-//
-// Sanitizes a selector
-//
-FORCEINLINE
-ULONG
-Ke386SanitizeSeg(IN ULONG Cs,
- IN KPROCESSOR_MODE Mode)
-{
- //
- // Check if we're in kernel-mode, and force CPL 0 if so.
- // Otherwise, force CPL 3.
- //
- return ((Mode == KernelMode) ?
- (Cs & (0xFFFF & ~RPL_MASK)) :
- (RPL_MASK | (Cs & 0xFFFF)));
-}
-
-//
-// Sanitizes EFLAGS
-//
-FORCEINLINE
-ULONG
-Ke386SanitizeFlags(IN ULONG Eflags,
- IN KPROCESSOR_MODE Mode)
-{
- //
- // Check if we're in kernel-mode, and sanitize EFLAGS if so.
- // Otherwise, also force interrupt mask on.
- //
- return ((Mode == KernelMode) ?
- (Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
- (EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
-}
-
-//
-// Gets a DR register from a CONTEXT structure
-//
-FORCEINLINE
-PVOID
-KiDrFromContext(IN ULONG Dr,
- IN PCONTEXT Context)
-{
- return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]);
-}
-
-//
-// Gets a DR register from a KTRAP_FRAME structure
-//
-FORCEINLINE
-PVOID*
-KiDrFromTrapFrame(IN ULONG Dr,
- IN PKTRAP_FRAME TrapFrame)
-{
- return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]);
-}
-
-//
-//
-//
-FORCEINLINE
-PVOID
-Ke386SanitizeDr(IN PVOID DrAddress,
- IN KPROCESSOR_MODE Mode)
-{
- //
- // Check if we're in kernel-mode, and return the address directly if so.
- // Otherwise, make sure it's not inside the kernel-mode address space.
- // If it is, then clear the address.
- //
- return ((Mode == KernelMode) ? DrAddress :
- (DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
-}
-#endif /* _M_IX86 */
-
#ifndef _M_ARM
-FORCEINLINE
-PRKTHREAD
-KeGetCurrentThread(VOID)
-{
-#ifdef _M_IX86
- /* Return the current thread */
- return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
-#elif defined (_M_AMD64)
- return (PRKTHREAD)__readgsqword(FIELD_OFFSET(KIPCR, Prcb.CurrentThread));
-#else
- PKPRCB Prcb = KeGetCurrentPrcb();
- return Prcb->CurrentThread;
-#endif
-}
-
FORCEINLINE
UCHAR
KeGetPreviousMode(VOID)
}
#endif
-FORCEINLINE
-VOID
-KeFlushProcessTb(VOID)
-{
- /* Flush the TLB by resetting CR3 */
-#ifdef _M_PPC
- __asm__("sync\n\tisync\n\t");
-#elif _M_ARM
- //
- // We need to implement this!
- //
- ASSERTMSG("Need ARM flush routine\n", FALSE);
-#else
- __writecr3(__readcr3());
-#endif
-}
-
//
// Enters a Guarded Region
//
} \
}
-//
-// TODO: Guarded Mutex Routines
-//
-
//
// Enters a Critical Region
//
}
#ifndef CONFIG_SMP
-//
-// Spinlock Acquire at IRQL >= DISPATCH_LEVEL
-//
-FORCEINLINE
-VOID
-KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
-{
- /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
- UNREFERENCED_PARAMETER(SpinLock);
-}
-
-//
-// Spinlock Release at IRQL >= DISPATCH_LEVEL
-//
-FORCEINLINE
-VOID
-KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
-{
- /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
- UNREFERENCED_PARAMETER(SpinLock);
-}
//
// This routine protects against multiple CPU acquires, it's meaningless on UP.
UNREFERENCED_PARAMETER(Prcb);
}
-FORCEINLINE
-VOID
-KiRundownThread(IN PKTHREAD Thread)
-{
-#if defined(_M_IX86)
- /* Check if this is the NPX Thread */
- if (KeGetCurrentPrcb()->NpxThread == Thread)
- {
- /* Clear it */
- KeGetCurrentPrcb()->NpxThread = NULL;
- KeArchFnInit();
- }
-#endif
-}
-
FORCEINLINE
VOID
KiRequestApcInterrupt(IN BOOLEAN NeedApc,
#else
-//
-// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
-//
-FORCEINLINE
-VOID
-KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
-{
- for (;;)
- {
- /* Try to acquire it */
- if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
- {
- /* Value changed... wait until it's locked */
- while (*(volatile KSPIN_LOCK *)SpinLock == 1)
- {
-#ifdef DBG
- /* On debug builds, we use a much slower but useful routine */
- //Kii386SpinOnSpinLock(SpinLock, 5);
-
- /* FIXME: Do normal yield for now */
- YieldProcessor();
-#else
- /* Otherwise, just yield and keep looping */
- YieldProcessor();
-#endif
- }
- }
- else
- {
-#ifdef DBG
- /* On debug builds, we OR in the KTHREAD */
- *SpinLock = (KSPIN_LOCK)KeGetCurrentThread() | 1;
-#endif
- /* All is well, break out */
- break;
- }
- }
-}
-
-//
-// Spinlock Release at IRQL >= DISPATCH_LEVEL
-//
-FORCEINLINE
-VOID
-KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
-{
-#ifdef DBG
- /* Make sure that the threads match */
- if (((KSPIN_LOCK)KeGetCurrentThread() | 1) != *SpinLock)
- {
- /* They don't, bugcheck */
- KeBugCheckEx(SPIN_LOCK_NOT_OWNED, (ULONG_PTR)SpinLock, 0, 0, 0);
- }
-#endif
- /* Clear the lock */
- InterlockedAnd((PLONG)SpinLock, 0);
-}
-
FORCEINLINE
VOID
KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object)
KiAcquireDispatcherLockAtDpcLevel(VOID)
{
/* Acquire the dispatcher lock */
- KeAcquireQueuedSpinLockAtDpcLevel(LockQueueDispatcherLock);
+ KeAcquireQueuedSpinLockAtDpcLevel(&KeGetCurrentPrcb()->
+ LockQueue[LockQueueDispatcherLock]);
}
FORCEINLINE
KiReleaseDispatcherLockFromDpcLevel(VOID)
{
/* Release the dispatcher lock */
- KeReleaseQueuedSpinLockFromDpcLevel(LockQueueDispatcherLock);
+ KeReleaseQueuedSpinLockFromDpcLevel(&KeGetCurrentPrcb()->
+ LockQueue[LockQueueDispatcherLock]);
}
//
-// This routine inserts a thread into the deferred ready list of the given CPU
+// This routine inserts a thread into the deferred ready list of the current CPU
//
FORCEINLINE
VOID
// This routine acquires the PRCB lock so that only one caller can touch
// volatile PRCB data.
//
-// Since this is a simple optimized spin-lock, it must be be only acquired
+// Since this is a simple optimized spin-lock, it must only be acquired
// at dispatcher level or higher!
//
FORCEINLINE
VOID
KiReleasePrcbLock(IN PKPRCB Prcb)
{
- /* Make sure it's acquired! */
+ /* Make sure we are above dispatch and the lock is acquired! */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
ASSERT(Prcb->PrcbLock != 0);
/* Release it */
VOID
KiReleaseThreadLock(IN PKTHREAD Thread)
{
+ /* Make sure we are still above dispatch */
+ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
+
/* Release it */
InterlockedAnd((PLONG)&Thread->ThreadLock, 0);
}
if (Prcb->DeferredReadyListHead.Next) KiProcessDeferredReadyList(Prcb);
}
-FORCEINLINE
-VOID
-KiRundownThread(IN PKTHREAD Thread)
-{
-#if defined(_M_IX86) || defined(_M_AMD64)
- /* FIXME: TODO */
- ASSERTMSG("Not yet implemented\n", FALSE);
-#endif
-}
-
FORCEINLINE
VOID
KiRequestApcInterrupt(IN BOOLEAN NeedApc,
return STATUS_WAIT_0;
}
+ULONG
+FORCEINLINE
+KiComputeTimerTableIndex(IN ULONGLONG DueTime)
+{
+ return (DueTime / KeMaximumIncrement) & (TIMER_TABLE_SIZE - 1);
+}
+
//
// Called from KiCompleteTimer, KiInsertTreeTimer, KeSetSystemTime
// to remove timer entries
/* Return the new priority */
return Priority;
}
+
+//
+// Guarded Mutex Routines
+//
+FORCEINLINE
+VOID
+_KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ /* Setup the Initial Data */
+ GuardedMutex->Count = GM_LOCK_BIT;
+ GuardedMutex->Owner = NULL;
+ GuardedMutex->Contention = 0;
+
+ /* Initialize the Wait Gate */
+ KeInitializeGate(&GuardedMutex->Gate);
+}
+
+FORCEINLINE
+VOID
+_KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ PKTHREAD Thread = KeGetCurrentThread();
+
+ /* Sanity checks */
+ ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+ (Thread->SpecialApcDisable < 0) ||
+ (Thread->Teb == NULL) ||
+ (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+ ASSERT(GuardedMutex->Owner != Thread);
+
+ /* Remove the lock */
+ if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+ {
+ /* The Guarded Mutex was already locked, enter contented case */
+ KiAcquireGuardedMutex(GuardedMutex);
+ }
+
+ /* Set the Owner */
+ GuardedMutex->Owner = Thread;
+}
+
+FORCEINLINE
+VOID
+_KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ LONG OldValue, NewValue;
+
+ /* Sanity checks */
+ ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+ (KeGetCurrentThread()->SpecialApcDisable < 0) ||
+ (KeGetCurrentThread()->Teb == NULL) ||
+ (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+ ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
+
+ /* Destroy the Owner */
+ GuardedMutex->Owner = NULL;
+
+ /* Add the Lock Bit */
+ OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
+ ASSERT((OldValue & GM_LOCK_BIT) == 0);
+
+ /* Check if it was already locked, but not woken */
+ if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
+ {
+ /* Update the Oldvalue to what it should be now */
+ OldValue += GM_LOCK_BIT;
+
+ /* The mutex will be woken, minus one waiter */
+ NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
+ GM_LOCK_WAITER_INC;
+
+ /* Remove the Woken bit */
+ if (InterlockedCompareExchange(&GuardedMutex->Count,
+ NewValue,
+ OldValue) == OldValue)
+ {
+ /* Signal the Gate */
+ KeSignalGateBoostPriority(&GuardedMutex->Gate);
+ }
+ }
+}
+
+FORCEINLINE
+VOID
+_KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
+{
+ PKTHREAD Thread = KeGetCurrentThread();
+
+ /* Sanity checks */
+ ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+ ASSERT(GuardedMutex->Owner != Thread);
+
+ /* Disable Special APCs */
+ KeEnterGuardedRegion();
+
+ /* Remove the lock */
+ if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+ {
+ /* The Guarded Mutex was already locked, enter contented case */
+ KiAcquireGuardedMutex(GuardedMutex);
+ }
+
+ /* Set the Owner and Special APC Disable state */
+ GuardedMutex->Owner = Thread;
+ GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
+}
+
+FORCEINLINE
+VOID
+_KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ LONG OldValue, NewValue;
+
+ /* Sanity checks */
+ ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+ ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
+ ASSERT(KeGetCurrentThread()->SpecialApcDisable ==
+ GuardedMutex->SpecialApcDisable);
+
+ /* Destroy the Owner */
+ GuardedMutex->Owner = NULL;
+
+ /* Add the Lock Bit */
+ OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
+ ASSERT((OldValue & GM_LOCK_BIT) == 0);
+
+ /* Check if it was already locked, but not woken */
+ if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
+ {
+ /* Update the Oldvalue to what it should be now */
+ OldValue += GM_LOCK_BIT;
+
+ /* The mutex will be woken, minus one waiter */
+ NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
+ GM_LOCK_WAITER_INC;
+
+ /* Remove the Woken bit */
+ if (InterlockedCompareExchange(&GuardedMutex->Count,
+ NewValue,
+ OldValue) == OldValue)
+ {
+ /* Signal the Gate */
+ KeSignalGateBoostPriority(&GuardedMutex->Gate);
+ }
+ }
+
+ /* Re-enable APCs */
+ KeLeaveGuardedRegion();
+}
+
+FORCEINLINE
+BOOLEAN
+_KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+ PKTHREAD Thread = KeGetCurrentThread();
+
+ /* Block APCs */
+ KeEnterGuardedRegion();
+
+ /* Remove the lock */
+ if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+ {
+ /* Re-enable APCs */
+ KeLeaveGuardedRegion();
+ YieldProcessor();
+
+ /* Return failure */
+ return FALSE;
+ }
+
+ /* Set the Owner and APC State */
+ GuardedMutex->Owner = Thread;
+ GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
+ return TRUE;
+}
+
+
+FORCEINLINE
+VOID
+KiAcquireNmiListLock(OUT PKIRQL OldIrql)
+{
+ KeAcquireSpinLock(&KiNmiCallbackListLock, OldIrql);
+}
+
+FORCEINLINE
+VOID
+KiReleaseNmiListLock(IN KIRQL OldIrql)
+{
+ KeReleaseSpinLock(&KiNmiCallbackListLock, OldIrql);
+}
projects / reactos.git / blobdiff