- Implement KxAcquireSpinLock/KxReleaseSpinLock for inline acquisition of spinlocks...

- Make In-Stack QSLs a complete no-op in UP, we were still touching some fields.
- Cleanup and re-organize spinlock.c

svn path=/trunk/; revision=24090

diff --git a/reactos/ntoskrnl/include/internal/ke_x.h b/reactos/ntoskrnl/include/internal/ke_x.h
index 5a5634c..c615c65 100644 (file)
--- a/reactos/ntoskrnl/include/internal/ke_x.h
+++ b/reactos/ntoskrnl/include/internal/ke_x.h
@@ -292,6 +292,71 @@ KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
     }                                                                       \\r
 }\r
 \r
+//\r
+// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL\r
+//\r
+FORCEINLINE\r
+VOID\r
+KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)\r
+{\r
+#ifdef CONFIG_SMP\r
+    for (;;)\r
+    {\r
+        /* Try to acquire it */\r
+        if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))\r
+        {\r
+            /* Value changed... wait until it's locked */\r
+            while (*(volatile KSPIN_LOCK *)SpinLock == 1)\r
+            {\r
+#ifdef DBG\r
+                /* On debug builds, we use a much slower but useful routine */\r
+                Kii386SpinOnSpinLock(SpinLock, 5);\r
+#else\r
+                /* Otherwise, just yield and keep looping */\r
+                YieldProcessor();\r
+#endif\r
+            }\r
+        }\r
+        else\r
+        {\r
+#ifdef DBG\r
+            /* On debug builds, we OR in the KTHREAD */\r
+            *SpinLock = KeGetCurrentThread() | 1;\r
+#endif\r
+            /* All is well, break out */\r
+            break;\r
+        }\r
+    }\r
+#else\r
+    /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */\r
+    UNREFERENCED_PARAMETER(SpinLock);\r
+#endif\r
+}\r
+\r
+//\r
+// Spinlock Release at IRQL >= DISPATCH_LEVEL\r
+//\r
+FORCEINLINE\r
+VOID\r
+KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)\r
+{\r
+#ifdef CONFIG_SMP\r
+#ifdef DBG\r
+    /* Make sure that the threads match */\r
+    if ((KeGetCurrentThread() | 1) != *SpinLock)\r
+    {\r
+        /* They don't, bugcheck */\r
+        KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0);\r
+    }\r
+#endif\r
+    /* Clear the lock */\r
+    InterlockedAnd(SpinLock, 0);\r
+#else\r
+    /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */\r
+    UNREFERENCED_PARAMETER(SpinLock);\r
+#endif\r
+}\r
+\r
 //\r
 // Thread Scheduling Routines\r
 //\r

diff --git a/reactos/ntoskrnl/ke/spinlock.c b/reactos/ntoskrnl/ke/spinlock.c
index 456576b..b78f5f1 100644 (file)
--- a/reactos/ntoskrnl/ke/spinlock.c
+++ b/reactos/ntoskrnl/ke/spinlock.c
@@ -1,117 +1,124 @@
 /*
- * COPYRIGHT:       See COPYING in the top level directory
  * PROJECT:         ReactOS Kernel
+ * LICENSE:         GPL - See COPYING in the top level directory
  * FILE:            ntoskrnl/ke/spinlock.c
- * PURPOSE:         Implements spinlocks
- * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
- *                  David Welch (welch@cwcom.net)
+ * PURPOSE:         Spinlock and Queued Spinlock Support
+ * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
  */
 
-/* INCLUDES ****************************************************************/
+/* INCLUDES ******************************************************************/
 
 #include <ntoskrnl.h>
 #define NDEBUG
 #include <internal/debug.h>
 
-#undef KefAcquireSpinLockAtDpcLevel
-#undef KeAcquireSpinLockAtDpcLevel
-#undef KefReleaseSpinLockFromDpcLevel
-#undef KeReleaseSpinLockFromDpcLevel
-
 #define LQ_WAIT     1
 #define LQ_OWN      2
 
-/* FUNCTIONS ***************************************************************/
+/* PRIVATE FUNCTIONS *********************************************************/
 
-/*
- * @implemented
- *
- * FUNCTION: Synchronizes the execution of a given routine with the ISR
- * of a given interrupt object
- * ARGUMENTS:
- *       Interrupt = Interrupt object to synchronize with
- *       SynchronizeRoutine = Routine to call whose execution is
- *                            synchronized with the ISR
- *       SynchronizeContext = Parameter to pass to the synchronized routine
- * RETURNS: TRUE if the operation succeeded
- */
-BOOLEAN
-STDCALL
-KeSynchronizeExecution(PKINTERRUPT Interrupt,
-                       PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
-                       PVOID SynchronizeContext)
+VOID
+FASTCALL
+KeAcquireQueuedSpinLockAtDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
-    KIRQL OldIrql;
-    BOOLEAN Status;
+#ifdef CONFIG_SMP
+    PKSPIN_LOCK_QUEUE Prev;
 
-    /* Raise IRQL and acquire lock on MP */
-    OldIrql = KeAcquireInterruptSpinLock(Interrupt);
+    /* Set the new lock */
+    Prev = (PKSPIN_LOCK_QUEUE)
+           InterlockedExchange((PLONG)LockHandle->LockQueue.Lock,
+                               (LONG)LockHandle);
+    if (!Prev)
+    {
+        /* There was nothing there before. We now own it */
+         *(ULONG_PTR*)&LockHandle->LockQueue.Lock |= LQ_OWN;
+        return;
+    }
 
-    /* Call the routine */
-    Status = SynchronizeRoutine(SynchronizeContext);
+    /* Set the wait flag */
+     *(ULONG_PTR*)&LockHandle->LockQueue.Lock |= LQ_WAIT;
 
-    /* Release lock and lower IRQL */
-    KeReleaseInterruptSpinLock(Interrupt, OldIrql);
+    /* Link us */
+    Prev->Next = (PKSPIN_LOCK_QUEUE)LockHandle;
 
-    /* Return routine status */
-    return Status;
+    /* Loop and wait */
+    while ( *(ULONG_PTR*)&LockHandle->LockQueue.Lock & LQ_WAIT) YieldProcessor();
+    return;
+#endif
 }
 
-/*
- * @implemented
- */
-KIRQL
-STDCALL
-KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
+VOID
+FASTCALL
+KeReleaseQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
-    KIRQL OldIrql;
+#ifdef CONFIG_SMP
+    KSPIN_LOCK LockVal;
+    PKSPIN_LOCK_QUEUE Waiter;
 
-    /* Raise IRQL */
-    KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
+    /* Remove own and wait flags */
+     *(ULONG_PTR*)&LockHandle->LockQueue.Lock &= ~(LQ_OWN | LQ_WAIT);
+    LockVal = *LockHandle->LockQueue.Lock;
 
-    /* Acquire spinlock on MP */
-    KiAcquireSpinLock(Interrupt->ActualLock);
-    return OldIrql;
+    /* Check if we already own it */
+    if (LockVal == (KSPIN_LOCK)LockHandle)
+    {
+        /* Disown it */
+        LockVal = (KSPIN_LOCK)
+                  InterlockedCompareExchangePointer(LockHandle->LockQueue.Lock,
+                                                    NULL,
+                                                    LockHandle);
+    }
+    if (LockVal == (KSPIN_LOCK)LockHandle) return;
+
+    /* Need to wait for it */
+    Waiter = LockHandle->LockQueue.Next;
+    while (!Waiter)
+    {
+        YieldProcessor();
+        Waiter = LockHandle->LockQueue.Next;
+    }
+
+    /* It's gone */
+    *(ULONG_PTR*)&Waiter->Lock ^= (LQ_OWN | LQ_WAIT);
+    LockHandle->LockQueue.Next = NULL;
+#endif
 }
 
+/* PUBLIC FUNCTIONS **********************************************************/
+
 /*
  * @implemented
- *
- * FUNCTION: Initalizes a spinlock
- * ARGUMENTS:
- *           SpinLock = Caller supplied storage for the spinlock
  */
 VOID
-STDCALL
-KeInitializeSpinLock(PKSPIN_LOCK SpinLock)
+NTAPI
+KeInitializeSpinLock(IN PKSPIN_LOCK SpinLock)
 {
+    /* Clear it */
     *SpinLock = 0;
 }
 
 /*
  * @implemented
  */
+#undef KeAcquireSpinLockAtDpcLevel
 VOID
-FASTCALL
-KefAcquireSpinLockAtDpcLevel(PKSPIN_LOCK SpinLock)
+NTAPI
+KeAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
 {
-    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-    KiAcquireSpinLock(SpinLock);
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
 }
 
 /*
  * @implemented
- *
- * FUNCTION: Acquires a spinlock when the caller is already running at
- * dispatch level
- * ARGUMENTS:
- *        SpinLock = Spinlock to acquire
  */
+#undef KeReleaseSpinLockFromDpcLevel
 VOID
-STDCALL
-KeAcquireSpinLockAtDpcLevel (PKSPIN_LOCK SpinLock)
+NTAPI
+KeReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
 {
-    KefAcquireSpinLockAtDpcLevel(SpinLock);
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
 }
 
 /*
@@ -119,25 +126,21 @@ KeAcquireSpinLockAtDpcLevel (PKSPIN_LOCK SpinLock)
  */
 VOID
 FASTCALL
-KefReleaseSpinLockFromDpcLevel(PKSPIN_LOCK SpinLock)
+KefAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
 {
-    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-    KiReleaseSpinLock(SpinLock);
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
 }
 
 /*
  * @implemented
- *
- * FUNCTION: Releases a spinlock when the caller was running at dispatch
- * level before acquiring it
- * ARGUMENTS:
- *         SpinLock = Spinlock to release
  */
 VOID
-STDCALL
-KeReleaseSpinLockFromDpcLevel (PKSPIN_LOCK SpinLock)
+FASTCALL
+KefReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
 {
-    KefReleaseSpinLockFromDpcLevel(SpinLock);
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
 }
 
 /*
@@ -145,39 +148,21 @@ KeReleaseSpinLockFromDpcLevel (PKSPIN_LOCK SpinLock)
  */
 VOID
 FASTCALL
-KiAcquireSpinLock(PKSPIN_LOCK SpinLock)
+KiAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
 {
-#ifdef CONFIG_SMP
-    for (;;)
-    {
-        /* Try to acquire it */
-        if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
-        {
-            /* Value changed... wait until it's locked */
-            while (*(volatile KSPIN_LOCK *)SpinLock == 1) YieldProcessor();
-        }
-        else
-        {
-            /* All is well, break out */
-            break;
-        }
-    }
-#endif /* CONFIG_SMP */
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
 }
 
 /*
  * @implemented
  */
 VOID
-STDCALL
-KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
-                           IN KIRQL OldIrql)
+FASTCALL
+KiReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
 {
-    /* Release lock on MP */
-    KiReleaseSpinLock(Interrupt->ActualLock);
-
-    /* Lower IRQL */
-    KeLowerIrql(OldIrql);
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
 }
 
 /*
@@ -185,106 +170,87 @@ KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
  */
 VOID
 FASTCALL
-KiReleaseSpinLock(PKSPIN_LOCK SpinLock)
+KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock,
+                                         IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
 #ifdef CONFIG_SMP
-    /* Simply clear it */
-    *SpinLock = 0;
+    /* Set it up properly */
+    LockHandle->LockQueue.Next = NULL;
+    LockHandle->LockQueue.Lock = SpinLock;
+    KeAcquireQueuedSpinLockAtDpcLevel((PKLOCK_QUEUE_HANDLE)
+                                      &LockHandle->LockQueue.Next);
 #endif
 }
 
+/*
+ * @implemented
+ */
 VOID
 FASTCALL
-KeAcquireQueuedSpinLockAtDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
+KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
 #ifdef CONFIG_SMP
-    PKSPIN_LOCK_QUEUE Prev;
-
-    /* Set the new lock */
-    Prev = (PKSPIN_LOCK_QUEUE)
-           InterlockedExchange((PLONG)LockHandle->LockQueue.Lock,
-                               (LONG)LockHandle);
-    if (!Prev)
-    {
-        /* There was nothing there before. We now own it */
-         *(ULONG_PTR*)&LockHandle->LockQueue.Lock |= LQ_OWN;
-        return;
-    }
-
-    /* Set the wait flag */
-     *(ULONG_PTR*)&LockHandle->LockQueue.Lock |= LQ_WAIT;
-
-    /* Link us */
-    Prev->Next = (PKSPIN_LOCK_QUEUE)LockHandle;
-
-    /* Loop and wait */
-    while ( *(ULONG_PTR*)&LockHandle->LockQueue.Lock & LQ_WAIT) YieldProcessor();
-    return;
+    /* Call the internal function */
+    KeReleaseQueuedSpinLockFromDpcLevel((PKLOCK_QUEUE_HANDLE)
+                                        &LockHandle->LockQueue.Next);
 #endif
 }
 
-VOID
-FASTCALL
-KeReleaseQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
+/*
+ * @implemented
+ */
+KIRQL
+NTAPI
+KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
 {
-#ifdef CONFIG_SMP
-    KSPIN_LOCK LockVal;
-    PKSPIN_LOCK_QUEUE Waiter;
-
-    /* Remove own and wait flags */
-     *(ULONG_PTR*)&LockHandle->LockQueue.Lock &= ~(LQ_OWN | LQ_WAIT);
-    LockVal = *LockHandle->LockQueue.Lock;
-
-    /* Check if we already own it */
-    if (LockVal == (KSPIN_LOCK)LockHandle)
-    {
-        /* Disown it */
-        LockVal = (KSPIN_LOCK)
-                  InterlockedCompareExchangePointer(LockHandle->LockQueue.Lock,
-                                                    NULL,
-                                                    LockHandle);
-    }
-    if (LockVal == (KSPIN_LOCK)LockHandle) return;
+    KIRQL OldIrql;
 
-    /* Need to wait for it */
-    Waiter = LockHandle->LockQueue.Next;
-    while (!Waiter)
-    {
-        YieldProcessor();
-        Waiter = LockHandle->LockQueue.Next;
-    }
+    /* Raise IRQL */
+    KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
 
-    /* It's gone */
-    *(ULONG_PTR*)&Waiter->Lock ^= (LQ_OWN | LQ_WAIT);
-    LockHandle->LockQueue.Next = NULL;
-#endif
+    /* Acquire spinlock on MP */
+    KefAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
+    return OldIrql;
 }
 
 /*
  * @implemented
  */
-VOID
-FASTCALL
-KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock,
-                                         IN PKLOCK_QUEUE_HANDLE LockHandle)
+BOOLEAN
+NTAPI
+KeSynchronizeExecution(IN PKINTERRUPT Interrupt,
+                       IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
+                       IN PVOID SynchronizeContext)
 {
-    /* Set it up properly */
-    LockHandle->LockQueue.Next = NULL;
-    LockHandle->LockQueue.Lock = SpinLock;
-    KeAcquireQueuedSpinLockAtDpcLevel((PKLOCK_QUEUE_HANDLE)
-                                      &LockHandle->LockQueue.Next);
+    KIRQL OldIrql;
+    BOOLEAN Status;
+
+    /* Raise IRQL and acquire lock on MP */
+    OldIrql = KeAcquireInterruptSpinLock(Interrupt);
+
+    /* Call the routine */
+    Status = SynchronizeRoutine(SynchronizeContext);
+
+    /* Release lock and lower IRQL */
+    KeReleaseInterruptSpinLock(Interrupt, OldIrql);
+
+    /* Return routine status */
+    return Status;
 }
 
 /*
  * @implemented
  */
 VOID
-FASTCALL
-KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
+STDCALL
+KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
+                           IN KIRQL OldIrql)
 {
-    /* Call the internal function */
-    KeReleaseQueuedSpinLockFromDpcLevel((PKLOCK_QUEUE_HANDLE)
-                                        &LockHandle->LockQueue.Next);
+    /* Release lock on MP */
+    KefReleaseSpinLock(Interrupt->ActualLock);
+
+    /* Lower IRQL */
+    KeLowerIrql(OldIrql);
 }
 
 /* EOF */