ntoskrnl/ke/spinlock.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/ke/spinlock.c
   5  * PURPOSE:         Spinlock and Queued Spinlock Support
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define LQ_WAIT     1
  16 #define LQ_OWN      2
  17
  18 /* PRIVATE FUNCTIONS *********************************************************/
  19
  20 #if 0
  21 //
  22 // FIXME: The queued spinlock routines are broken.
  23 //
  24
  25 VOID
  26 FASTCALL
  27 KeAcquireQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK_QUEUE LockHandle)
  28 {
  29 #ifdef CONFIG_SMP
  30     PKSPIN_LOCK_QUEUE Prev;
  31
  32     /* Set the new lock */
  33     Prev = (PKSPIN_LOCK_QUEUE)
  34            InterlockedExchange((PLONG)LockHandle->Next,
  35                                (LONG)LockHandle);
  36     if (!Prev)
  37     {
  38         /* There was nothing there before. We now own it */
  39          *LockHandle->Lock |= LQ_OWN;
  40         return;
  41     }
  42
  43     /* Set the wait flag */
  44      *LockHandle->Lock |= LQ_WAIT;
  45
  46     /* Link us */
  47     Prev->Next = (PKSPIN_LOCK_QUEUE)LockHandle;
  48
  49     /* Loop and wait */
  50     while (*LockHandle->Lock & LQ_WAIT)
  51         YieldProcessor();
  52 #endif
  53 }
  54
  55 VOID
  56 FASTCALL
  57 KeReleaseQueuedSpinLockFromDpcLevel(IN PKSPIN_LOCK_QUEUE LockHandle)
  58 {
  59 #ifdef CONFIG_SMP
  60     KSPIN_LOCK LockVal;
  61     PKSPIN_LOCK_QUEUE Waiter;
  62
  63     /* Remove own and wait flags */
  64     *LockHandle->Lock &= ~(LQ_OWN | LQ_WAIT);
  65     LockVal = *LockHandle->Lock;
  66
  67     /* Check if we already own it */
  68     if (LockVal == (KSPIN_LOCK)LockHandle)
  69     {
  70         /* Disown it */
  71         LockVal = (KSPIN_LOCK)
  72                   InterlockedCompareExchangePointer(LockHandle->Lock,
  73                                                     NULL,
  74                                                     LockHandle);
  75     }
  76     if (LockVal == (KSPIN_LOCK)LockHandle) return;
  77
  78     /* Need to wait for it */
  79     Waiter = LockHandle->Next;
  80     while (!Waiter)
  81     {
  82         YieldProcessor();
  83         Waiter = LockHandle->Next;
  84     }
  85
  86     /* It's gone */
  87     *(ULONG_PTR*)&Waiter->Lock ^= (LQ_OWN | LQ_WAIT);
  88     LockHandle->Next = NULL;
  89 #endif
  90 }
  91
  92 #else
  93 //
  94 // HACK: Hacked to work like normal spinlocks
  95 //
  96
  97 _IRQL_requires_min_(DISPATCH_LEVEL)
  98 _Acquires_nonreentrant_lock_(*LockHandle->Lock)
  99 _Acquires_exclusive_lock_(*LockHandle->Lock)
 100 VOID
 101 FASTCALL
 102 KeAcquireQueuedSpinLockAtDpcLevel(_Inout_ PKSPIN_LOCK_QUEUE LockHandle)
 103 {
 104 #if defined(CONFIG_SMP) || DBG
 105     /* Make sure we are at DPC or above! */
 106     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 107     {
 108         /* We aren't -- bugcheck */
 109         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 110                      (ULONG_PTR)LockHandle->Lock,
 111                      KeGetCurrentIrql(),
 112                      0,
 113                      0);
 114     }
 115 #endif
 116
 117     /* Do the inlined function */
 118     KxAcquireSpinLock(LockHandle->Lock);
 119 }
 120
 121 _IRQL_requires_min_(DISPATCH_LEVEL)
 122 _Releases_nonreentrant_lock_(*LockHandle->Lock)
 123 _Releases_exclusive_lock_(*LockHandle->Lock)
 124 VOID
 125 FASTCALL
 126 KeReleaseQueuedSpinLockFromDpcLevel(_Inout_ PKSPIN_LOCK_QUEUE LockHandle)
 127 {
 128 #if defined(CONFIG_SMP) || DBG
 129     /* Make sure we are at DPC or above! */
 130     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 131     {
 132         /* We aren't -- bugcheck */
 133         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 134                      (ULONG_PTR)LockHandle->Lock,
 135                      KeGetCurrentIrql(),
 136                      0,
 137                      0);
 138     }
 139 #endif
 140
 141     /* Do the inlined function */
 142     KxReleaseSpinLock(LockHandle->Lock);
 143 }
 144
 145 #endif
 146
 147 /* PUBLIC FUNCTIONS **********************************************************/
 148
 149 /*
 150  * @implemented
 151  */
 152 KIRQL
 153 NTAPI
 154 KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
 155 {
 156     KIRQL OldIrql;
 157
 158     /* Raise IRQL */
 159     KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
 160
 161     /* Acquire spinlock on MP */
 162     KeAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
 163     return OldIrql;
 164 }
 165
 166 /*
 167  * @implemented
 168  */
 169 VOID
 170 NTAPI
 171 KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
 172                            IN KIRQL OldIrql)
 173 {
 174     /* Release lock on MP */
 175     KeReleaseSpinLockFromDpcLevel(Interrupt->ActualLock);
 176
 177     /* Lower IRQL */
 178     KeLowerIrql(OldIrql);
 179 }
 180
 181 /*
 182  * @implemented
 183  */
 184 VOID
 185 NTAPI
 186 _KeInitializeSpinLock(IN PKSPIN_LOCK SpinLock)
 187 {
 188     /* Clear it */
 189     *SpinLock = 0;
 190 }
 191
 192 /*
 193  * @implemented
 194  */
 195 #undef KeAcquireSpinLockAtDpcLevel
 196 VOID
 197 NTAPI
 198 KeAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
 199 {
 200     /* Make sure we are at DPC or above! */
 201     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 202     {
 203         /* We aren't -- bugcheck */
 204         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 205                      (ULONG_PTR)SpinLock,
 206                      KeGetCurrentIrql(),
 207                      0,
 208                      0);
 209     }
 210
 211     /* Do the inlined function */
 212     KxAcquireSpinLock(SpinLock);
 213 }
 214
 215 /*
 216  * @implemented
 217  */
 218 #undef KeReleaseSpinLockFromDpcLevel
 219 VOID
 220 NTAPI
 221 KeReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
 222 {
 223     /* Make sure we are at DPC or above! */
 224     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 225     {
 226         /* We aren't -- bugcheck */
 227         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 228                      (ULONG_PTR)SpinLock,
 229                      KeGetCurrentIrql(),
 230                      0,
 231                      0);
 232     }
 233
 234     /* Do the inlined function */
 235     KxReleaseSpinLock(SpinLock);
 236 }
 237
 238 /*
 239  * @implemented
 240  */
 241 VOID
 242 FASTCALL
 243 KefAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
 244 {
 245     /* Make sure we are at DPC or above! */
 246     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 247     {
 248         /* We aren't -- bugcheck */
 249         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 250                      (ULONG_PTR)SpinLock,
 251                      KeGetCurrentIrql(),
 252                      0,
 253                      0);
 254     }
 255
 256     /* Do the inlined function */
 257     KxAcquireSpinLock(SpinLock);
 258 }
 259
 260 /*
 261  * @implemented
 262  */
 263 VOID
 264 FASTCALL
 265 KefReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
 266 {
 267     /* Make sure we are at DPC or above! */
 268     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 269     {
 270         /* We aren't -- bugcheck */
 271         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 272                      (ULONG_PTR)SpinLock,
 273                      KeGetCurrentIrql(),
 274                      0,
 275                      0);
 276     }
 277
 278     /* Do the inlined function */
 279     KxReleaseSpinLock(SpinLock);
 280 }
 281
 282 /*
 283  * @implemented
 284  */
 285 VOID
 286 FASTCALL
 287 KiAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
 288 {
 289     /* Do the inlined function */
 290     KxAcquireSpinLock(SpinLock);
 291 }
 292
 293 /*
 294  * @implemented
 295  */
 296 VOID
 297 FASTCALL
 298 KiReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
 299 {
 300     /* Do the inlined function */
 301     KxReleaseSpinLock(SpinLock);
 302 }
 303
 304 /*
 305  * @implemented
 306  */
 307 BOOLEAN
 308 FASTCALL
 309 KeTryToAcquireSpinLockAtDpcLevel(IN OUT PKSPIN_LOCK SpinLock)
 310 {
 311 #if DBG
 312     /* Make sure we are at DPC or above! */
 313     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 314     {
 315         /* We aren't -- bugcheck */
 316         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 317                      (ULONG_PTR)SpinLock,
 318                      KeGetCurrentIrql(),
 319                      0,
 320                      0);
 321     }
 322
 323     /* Make sure that we don't own the lock already */
 324     if (((KSPIN_LOCK)KeGetCurrentThread() | 1) == *SpinLock)
 325     {
 326         /* We do, bugcheck! */
 327         KeBugCheckEx(SPIN_LOCK_ALREADY_OWNED, (ULONG_PTR)SpinLock, 0, 0, 0);
 328     }
 329 #endif
 330
 331 #ifdef CONFIG_SMP
 332     /* Check if it's already acquired */
 333     if (!(*SpinLock))
 334     {
 335         /* Try to acquire it */
 336         if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
 337         {
 338             /* Someone else acquired it */
 339             return FALSE;
 340         }
 341     }
 342     else
 343     {
 344         /* It was already acquired */
 345         return FALSE;
 346     }
 347 #endif
 348
 349 #if DBG
 350     /* On debug builds, we OR in the KTHREAD */
 351     *SpinLock = (ULONG_PTR)KeGetCurrentThread() | 1;
 352 #endif
 353
 354     /* All is well, return TRUE */
 355     return TRUE;
 356 }
 357
 358 /*
 359  * @implemented
 360  */
 361 VOID
 362 FASTCALL
 363 KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock,
 364                                          IN PKLOCK_QUEUE_HANDLE LockHandle)
 365 {
 366     /* Set it up properly */
 367     LockHandle->LockQueue.Next = NULL;
 368     LockHandle->LockQueue.Lock = SpinLock;
 369 #ifdef CONFIG_SMP
 370 #if 0
 371     KeAcquireQueuedSpinLockAtDpcLevel(LockHandle->LockQueue.Next);
 372 #else
 373     /* Make sure we are at DPC or above! */
 374     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 375     {
 376         /* We aren't -- bugcheck */
 377         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 378                      (ULONG_PTR)LockHandle->LockQueue.Lock,
 379                      KeGetCurrentIrql(),
 380                      0,
 381                      0);
 382     }
 383 #endif
 384 #endif
 385
 386     /* Acquire the lock */
 387     KxAcquireSpinLock(LockHandle->LockQueue.Lock); // HACK
 388 }
 389
 390 /*
 391  * @implemented
 392  */
 393 VOID
 394 FASTCALL
 395 KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 396 {
 397 #ifdef CONFIG_SMP
 398 #if 0
 399     /* Call the internal function */
 400     KeReleaseQueuedSpinLockFromDpcLevel(LockHandle->LockQueue.Next);
 401 #else
 402     /* Make sure we are at DPC or above! */
 403     if (KeGetCurrentIrql() < DISPATCH_LEVEL)
 404     {
 405         /* We aren't -- bugcheck */
 406         KeBugCheckEx(IRQL_NOT_GREATER_OR_EQUAL,
 407                      (ULONG_PTR)LockHandle->LockQueue.Lock,
 408                      KeGetCurrentIrql(),
 409                      0,
 410                      0);
 411     }
 412 #endif
 413 #endif
 414
 415     /* Release the lock */
 416     KxReleaseSpinLock(LockHandle->LockQueue.Lock); // HACK
 417 }
 418
 419 /*
 420  * @unimplemented
 421  */
 422 KIRQL
 423 FASTCALL
 424 KeAcquireSpinLockForDpc(IN PKSPIN_LOCK SpinLock)
 425 {
 426     UNIMPLEMENTED;
 427     return 0;
 428 }
 429
 430 /*
 431  * @unimplemented
 432  */
 433 VOID
 434 FASTCALL
 435 KeReleaseSpinLockForDpc(IN PKSPIN_LOCK SpinLock,
 436                         IN KIRQL OldIrql)
 437 {
 438     UNIMPLEMENTED;
 439 }
 440
 441 /*
 442  * @implemented
 443  */
 444 VOID
 445 FASTCALL
 446 KeAcquireInStackQueuedSpinLockForDpc(IN PKSPIN_LOCK SpinLock,
 447                                      IN PKLOCK_QUEUE_HANDLE LockHandle)
 448 {
 449     LockHandle->OldIrql = KeGetCurrentIrql();
 450     if (LockHandle->OldIrql >= DISPATCH_LEVEL)
 451         KeAcquireInStackQueuedSpinLockAtDpcLevel(SpinLock, LockHandle);
 452     else
 453         KeAcquireInStackQueuedSpinLock(SpinLock, LockHandle);
 454 }
 455
 456 /*
 457  * @implemented
 458  */
 459 VOID
 460 FASTCALL
 461 KeReleaseInStackQueuedSpinLockForDpc(IN PKLOCK_QUEUE_HANDLE LockHandle)
 462 {
 463     if (LockHandle->OldIrql >= DISPATCH_LEVEL)
 464         KeReleaseInStackQueuedSpinLockFromDpcLevel(LockHandle);
 465     else
 466         KeReleaseInStackQueuedSpinLock(LockHandle);
 467
 468 }
 469
 470 /*
 471  * @implemented
 472  */
 473 BOOLEAN
 474 FASTCALL
 475 KeTestSpinLock(IN PKSPIN_LOCK SpinLock)
 476 {
 477     /* Test this spinlock */
 478     if (*SpinLock)
 479     {
 480         /* Spinlock is busy, yield execution */
 481         YieldProcessor();
 482
 483         /* Return busy flag */
 484         return FALSE;
 485     }
 486
 487     /* Spinlock appears to be free */
 488     return TRUE;
 489 }
 490
 491 #ifdef _M_IX86
 492 VOID
 493 NTAPI
 494 Kii386SpinOnSpinLock(PKSPIN_LOCK SpinLock, ULONG Flags)
 495 {
 496     // FIXME: Handle flags
 497     UNREFERENCED_PARAMETER(Flags);
 498
 499     /* Spin until it's unlocked */
 500     while (*(volatile KSPIN_LOCK *)SpinLock & 1)
 501     {
 502         // FIXME: Check for timeout
 503
 504         /* Yield and keep looping */
 505         YieldProcessor();
 506     }
 507 }
 508 #endif