lib/rtl/srw.c

   1 /*
   2  * COPYRIGHT:         See COPYING in the top level directory
   3  * PROJECT:           ReactOS system libraries
   4  * PURPOSE:           Slim Reader/Writer (SRW) Routines
   5  * PROGRAMMER:        Thomas Weidenmueller <w3seek@reactos.com>
   6  *
   7  * NOTES:             The algorithms used in this implementation
   8  *                    may be different from Vista's implementation.
   9  *                    Since applications should treat the RTL_SRWLOCK
  10  *                    structure as opaque data, it should not matter.
  11  *                    The algorithms are probably not as optimized.
  12  */
  13
  14 /* INCLUDES *****************************************************************/
  15
  16 #include <rtl.h>
  17
  18 #define NDEBUG
  19 #include <debug.h>
  20
  21 /* FUNCTIONS *****************************************************************/
  22
  23 #ifdef _WIN64
  24 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet64((PLONGLONG)ptr,(LONGLONG)val)
  25 #define InterlockedAddPointer(ptr,val) InterlockedAdd64((PLONGLONG)ptr,(LONGLONG)val)
  26 #define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)
  27 #define InterlockedOrPointer(ptr,val) InterlockedOr64((PLONGLONG)ptr,(LONGLONG)val)
  28 #else
  29 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet((PLONG)ptr,(LONG)val)
  30 #define InterlockedAddPointer(ptr,val) InterlockedAdd((PLONG)ptr,(LONG)val)
  31 #define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)
  32 #define InterlockedOrPointer(ptr,val) InterlockedOr((PLONG)ptr,(LONG)val)
  33 #endif
  34
  35 #define RTL_SRWLOCK_OWNED_BIT   0
  36 #define RTL_SRWLOCK_CONTENDED_BIT   1
  37 #define RTL_SRWLOCK_SHARED_BIT  2
  38 #define RTL_SRWLOCK_CONTENTION_LOCK_BIT 3
  39 #define RTL_SRWLOCK_OWNED   (1 << RTL_SRWLOCK_OWNED_BIT)
  40 #define RTL_SRWLOCK_CONTENDED   (1 << RTL_SRWLOCK_CONTENDED_BIT)
  41 #define RTL_SRWLOCK_SHARED  (1 << RTL_SRWLOCK_SHARED_BIT)
  42 #define RTL_SRWLOCK_CONTENTION_LOCK (1 << RTL_SRWLOCK_CONTENTION_LOCK_BIT)
  43 #define RTL_SRWLOCK_MASK    (RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED | \
  44                              RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENTION_LOCK)
  45 #define RTL_SRWLOCK_BITS    4
  46
  47 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
  48     (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
  49 /* This macro will cause the code to assert if compiled with a buggy
  50    version of GCC that doesn't align the wait blocks properly on the stack! */
  51 #define ASSERT_SRW_WAITBLOCK(ptr) \
  52     ASSERT(((ULONG_PTR)ptr & ((1 << RTL_SRWLOCK_BITS) - 1)) == 0)
  53 #else
  54 #define ASSERT_SRW_WAITBLOCK(ptr) ((void)0)
  55 #endif
  56
  57 typedef struct _RTLP_SRWLOCK_SHARED_WAKE
  58 {
  59     LONG Wake;
  60     volatile struct _RTLP_SRWLOCK_SHARED_WAKE *Next;
  61 } volatile RTLP_SRWLOCK_SHARED_WAKE, *PRTLP_SRWLOCK_SHARED_WAKE;
  62
  63 typedef struct _RTLP_SRWLOCK_WAITBLOCK
  64 {
  65     /* SharedCount is the number of shared acquirers. */
  66     LONG SharedCount;
  67
  68     /* Last points to the last wait block in the chain. The value
  69        is only valid when read from the first wait block. */
  70     volatile struct _RTLP_SRWLOCK_WAITBLOCK *Last;
  71
  72     /* Next points to the next wait block in the chain. */
  73     volatile struct _RTLP_SRWLOCK_WAITBLOCK *Next;
  74
  75     union
  76     {
  77         /* Wake is only valid for exclusive wait blocks */
  78         LONG Wake;
  79         /* The wake chain is only valid for shared wait blocks */
  80         struct
  81         {
  82             PRTLP_SRWLOCK_SHARED_WAKE SharedWakeChain;
  83             PRTLP_SRWLOCK_SHARED_WAKE LastSharedWake;
  84         };
  85     };
  86
  87     BOOLEAN Exclusive;
  88 } volatile RTLP_SRWLOCK_WAITBLOCK, *PRTLP_SRWLOCK_WAITBLOCK;
  89
  90
  91 static VOID
  92 NTAPI
  93 RtlpReleaseWaitBlockLockExclusive(IN OUT PRTL_SRWLOCK SRWLock,
  94                                   IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
  95 {
  96     PRTLP_SRWLOCK_WAITBLOCK Next;
  97     LONG_PTR NewValue;
  98
  99     /* NOTE: We're currently in an exclusive lock in contended mode. */
 100
 101     Next = FirstWaitBlock->Next;
 102     if (Next != NULL)
 103     {
 104         /* There's more blocks chained, we need to update the pointers
 105            in the next wait block and update the wait block pointer. */
 106         NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 107         if (!FirstWaitBlock->Exclusive)
 108         {
 109             /* The next wait block has to be an exclusive lock! */
 110             ASSERT(Next->Exclusive);
 111
 112             /* Save the shared count */
 113             Next->SharedCount = FirstWaitBlock->SharedCount;
 114
 115             NewValue |= RTL_SRWLOCK_SHARED;
 116         }
 117
 118         Next->Last = FirstWaitBlock->Last;
 119     }
 120     else
 121     {
 122         /* Convert the lock to a simple lock. */
 123         if (FirstWaitBlock->Exclusive)
 124             NewValue = RTL_SRWLOCK_OWNED;
 125         else
 126         {
 127             ASSERT(FirstWaitBlock->SharedCount > 0);
 128
 129             NewValue = ((LONG_PTR)FirstWaitBlock->SharedCount << RTL_SRWLOCK_BITS) |
 130                        RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 131         }
 132     }
 133
 134     (void)_InterlockedExchange((PLONG)&SRWLock->Ptr, NewValue);
 135
 136     if (FirstWaitBlock->Exclusive)
 137     {
 138         (void)InterlockedOr(&FirstWaitBlock->Wake,
 139                             TRUE);
 140     }
 141     else
 142     {
 143         PRTLP_SRWLOCK_SHARED_WAKE WakeChain, Next;
 144
 145         /* If we were the first one to acquire the shared
 146            lock, we now need to wake all others... */
 147         WakeChain = FirstWaitBlock->SharedWakeChain;
 148         do
 149         {
 150             Next = WakeChain->Next;
 151
 152             (void)InterlockedOr((PLONG)&WakeChain->Wake,
 153                                 TRUE);
 154
 155             WakeChain = Next;
 156         } while (WakeChain != NULL);
 157     }
 158 }
 159
 160
 161 static VOID
 162 NTAPI
 163 RtlpReleaseWaitBlockLockLastShared(IN OUT PRTL_SRWLOCK SRWLock,
 164                                    IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
 165 {
 166     PRTLP_SRWLOCK_WAITBLOCK Next;
 167     LONG_PTR NewValue;
 168
 169     /* NOTE: We're currently in a shared lock in contended mode. */
 170
 171     /* The next acquirer to be unwaited *must* be an exclusive lock! */
 172     ASSERT(FirstWaitBlock->Exclusive);
 173
 174     Next = FirstWaitBlock->Next;
 175     if (Next != NULL)
 176     {
 177         /* There's more blocks chained, we need to update the pointers
 178            in the next wait block and update the wait block pointer. */
 179         NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 180
 181         Next->Last = FirstWaitBlock->Last;
 182     }
 183     else
 184     {
 185         /* Convert the lock to a simple exclusive lock. */
 186         NewValue = RTL_SRWLOCK_OWNED;
 187     }
 188
 189     (void)_InterlockedExchange((PLONG)&SRWLock->Ptr, NewValue);
 190
 191     (void)InterlockedOr(&FirstWaitBlock->Wake,
 192                         TRUE);
 193 }
 194
 195
 196 static VOID
 197 NTAPI
 198 RtlpReleaseWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
 199 {
 200     InterlockedAndPointer(&SRWLock->Ptr,
 201                           ~RTL_SRWLOCK_CONTENTION_LOCK);
 202 }
 203
 204
 205 static PRTLP_SRWLOCK_WAITBLOCK
 206 NTAPI
 207 RtlpAcquireWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
 208 {
 209     LONG_PTR PrevValue;
 210     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 211
 212     while (1)
 213     {
 214         PrevValue = InterlockedOrPointer(&SRWLock->Ptr,
 215                                          RTL_SRWLOCK_CONTENTION_LOCK);
 216
 217         if (!(PrevValue & RTL_SRWLOCK_CONTENTION_LOCK))
 218             break;
 219
 220         YieldProcessor();
 221     }
 222
 223     if (!(PrevValue & RTL_SRWLOCK_CONTENDED) ||
 224         (PrevValue & ~RTL_SRWLOCK_MASK) == 0)
 225     {
 226         /* Too bad, looks like the wait block was removed in the
 227            meanwhile, unlock again */
 228         RtlpReleaseWaitBlockLock(SRWLock);
 229         return NULL;
 230     }
 231
 232     WaitBlock = (PRTLP_SRWLOCK_WAITBLOCK)(PrevValue & ~RTL_SRWLOCK_MASK);
 233
 234     ASSERT_SRW_WAITBLOCK(WaitBlock);
 235     return WaitBlock;
 236 }
 237
 238
 239 static VOID
 240 NTAPI
 241 RtlpAcquireSRWLockExclusiveWait(IN OUT PRTL_SRWLOCK SRWLock,
 242                                 IN PRTLP_SRWLOCK_WAITBLOCK WaitBlock)
 243 {
 244     LONG_PTR CurrentValue;
 245
 246     while (1)
 247     {
 248         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 249         if (!(CurrentValue & RTL_SRWLOCK_SHARED))
 250         {
 251             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 252             {
 253                 if (WaitBlock->Wake != 0)
 254                 {
 255                     /* Our wait block became the first one
 256                        in the chain, we own the lock now! */
 257                     break;
 258                 }
 259             }
 260             else
 261             {
 262                 /* The last wait block was removed and/or we're
 263                    finally a simple exclusive lock. This means we
 264                    don't need to wait anymore, we acquired the lock! */
 265                 break;
 266             }
 267         }
 268
 269         YieldProcessor();
 270     }
 271 }
 272
 273
 274 static VOID
 275 NTAPI
 276 RtlpAcquireSRWLockSharedWait(IN OUT PRTL_SRWLOCK SRWLock,
 277                              IN OUT PRTLP_SRWLOCK_WAITBLOCK FirstWait  OPTIONAL,
 278                              IN OUT PRTLP_SRWLOCK_SHARED_WAKE WakeChain)
 279 {
 280     if (FirstWait != NULL)
 281     {
 282         while (WakeChain->Wake == 0)
 283         {
 284             YieldProcessor();
 285         }
 286     }
 287     else
 288     {
 289         LONG_PTR CurrentValue;
 290
 291         while (1)
 292         {
 293             CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 294             if (CurrentValue & RTL_SRWLOCK_SHARED)
 295             {
 296                 /* The RTL_SRWLOCK_OWNED bit always needs to be set when
 297                    RTL_SRWLOCK_SHARED is set! */
 298                 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
 299
 300                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 301                 {
 302                     if (WakeChain->Wake != 0)
 303                     {
 304                         /* Our wait block became the first one
 305                            in the chain, we own the lock now! */
 306                         break;
 307                     }
 308                 }
 309                 else
 310                 {
 311                     /* The last wait block was removed and/or we're
 312                        finally a simple shared lock. This means we
 313                        don't need to wait anymore, we acquired the lock! */
 314                     break;
 315                 }
 316             }
 317
 318             YieldProcessor();
 319         }
 320     }
 321 }
 322
 323
 324 VOID
 325 NTAPI
 326 RtlInitializeSRWLock(OUT PRTL_SRWLOCK SRWLock)
 327 {
 328     SRWLock->Ptr = NULL;
 329 }
 330
 331
 332 VOID
 333 NTAPI
 334 RtlAcquireSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
 335 {
 336     __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
 337     RTLP_SRWLOCK_SHARED_WAKE SharedWake;
 338     LONG_PTR CurrentValue, NewValue;
 339     PRTLP_SRWLOCK_WAITBLOCK First, Shared, FirstWait;
 340
 341     while (1)
 342     {
 343         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 344
 345         if (CurrentValue & RTL_SRWLOCK_SHARED)
 346         {
 347             /* NOTE: It is possible that the RTL_SRWLOCK_OWNED bit is set! */
 348
 349             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 350             {
 351                 /* There's other waiters already, lock the wait blocks and
 352                    increment the shared count */
 353                 First = RtlpAcquireWaitBlockLock(SRWLock);
 354                 if (First != NULL)
 355                 {
 356                     FirstWait = NULL;
 357
 358                     if (First->Exclusive)
 359                     {
 360                         /* We need to setup a new wait block! Although
 361                            we're currently in a shared lock and we're acquiring
 362                            a shared lock, there are exclusive locks queued. We need
 363                            to wait until those are released. */
 364                         Shared = First->Last;
 365
 366                         if (Shared->Exclusive)
 367                         {
 368                             StackWaitBlock.Exclusive = FALSE;
 369                             StackWaitBlock.SharedCount = 1;
 370                             StackWaitBlock.Next = NULL;
 371                             StackWaitBlock.Last = &StackWaitBlock;
 372                             StackWaitBlock.SharedWakeChain = &SharedWake;
 373
 374                             Shared->Next = &StackWaitBlock;
 375                             First->Last = &StackWaitBlock;
 376
 377                             Shared = &StackWaitBlock;
 378                             FirstWait = &StackWaitBlock;
 379                         }
 380                         else
 381                         {
 382                             Shared->LastSharedWake->Next = &SharedWake;
 383                             Shared->SharedCount++;
 384                         }
 385                     }
 386                     else
 387                     {
 388                         Shared = First;
 389                         Shared->LastSharedWake->Next = &SharedWake;
 390                         Shared->SharedCount++;
 391                     }
 392
 393                     SharedWake.Next = NULL;
 394                     SharedWake.Wake = 0;
 395
 396                     Shared->LastSharedWake = &SharedWake;
 397
 398                     ASSERT_SRW_WAITBLOCK(Shared);
 399
 400                     RtlpReleaseWaitBlockLock(SRWLock);
 401
 402                     RtlpAcquireSRWLockSharedWait(SRWLock,
 403                                                  FirstWait,
 404                                                  &SharedWake);
 405
 406                     /* Successfully incremented the shared count, we acquired the lock */
 407                     break;
 408                 }
 409             }
 410             else
 411             {
 412                 /* This is a fastest path, just increment the number of
 413                    current shared locks */
 414
 415                 /* Since the RTL_SRWLOCK_SHARED bit is set, the RTL_SRWLOCK_OWNED bit also has
 416                    to be set! */
 417
 418                 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
 419
 420                 NewValue = (CurrentValue >> RTL_SRWLOCK_BITS) + 1;
 421                 NewValue = (NewValue << RTL_SRWLOCK_BITS) | (CurrentValue & RTL_SRWLOCK_MASK);
 422
 423                 if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 424                                                 NewValue,
 425                                                 CurrentValue) == CurrentValue)
 426                 {
 427                     /* Successfully incremented the shared count, we acquired the lock */
 428                     break;
 429                 }
 430             }
 431         }
 432         else
 433         {
 434             if (CurrentValue & RTL_SRWLOCK_OWNED)
 435             {
 436                 /* The resource is currently acquired exclusively */
 437                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 438                 {
 439                     SharedWake.Next = NULL;
 440                     SharedWake.Wake = 0;
 441
 442                     /* There's other waiters already, lock the wait blocks and
 443                        increment the shared count. If the last block in the chain
 444                        is an exclusive lock, add another block. */
 445
 446                     StackWaitBlock.Exclusive = FALSE;
 447                     StackWaitBlock.SharedCount = 0;
 448                     StackWaitBlock.Next = NULL;
 449                     StackWaitBlock.Last = &StackWaitBlock;
 450                     StackWaitBlock.SharedWakeChain = &SharedWake;
 451
 452                     First = RtlpAcquireWaitBlockLock(SRWLock);
 453                     if (First != NULL)
 454                     {
 455                         Shared = First->Last;
 456                         if (Shared->Exclusive)
 457                         {
 458                             Shared->Next = &StackWaitBlock;
 459                             First->Last = &StackWaitBlock;
 460
 461                             Shared = &StackWaitBlock;
 462                             FirstWait = &StackWaitBlock;
 463                         }
 464                         else
 465                         {
 466                             FirstWait = NULL;
 467                             Shared->LastSharedWake->Next = &SharedWake;
 468                         }
 469
 470                         ASSERT_SRW_WAITBLOCK(Shared);
 471
 472                         Shared->SharedCount++;
 473                         Shared->LastSharedWake = &SharedWake;
 474
 475                         RtlpReleaseWaitBlockLock(SRWLock);
 476
 477                         RtlpAcquireSRWLockSharedWait(SRWLock,
 478                                                      FirstWait,
 479                                                      &SharedWake);
 480
 481                         /* Successfully incremented the shared count, we acquired the lock */
 482                         break;
 483                     }
 484                 }
 485                 else
 486                 {
 487                     SharedWake.Next = NULL;
 488                     SharedWake.Wake = 0;
 489
 490                     /* We need to setup the first wait block. Currently an exclusive lock is
 491                        held, change the lock to contended mode. */
 492                     StackWaitBlock.Exclusive = FALSE;
 493                     StackWaitBlock.SharedCount = 1;
 494                     StackWaitBlock.Next = NULL;
 495                     StackWaitBlock.Last = &StackWaitBlock;
 496                     StackWaitBlock.SharedWakeChain = &SharedWake;
 497                     StackWaitBlock.LastSharedWake = &SharedWake;
 498
 499                     ASSERT_SRW_WAITBLOCK(&StackWaitBlock);
 500
 501                     NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 502                     if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 503                                                     NewValue,
 504                                                     CurrentValue) == CurrentValue)
 505                     {
 506                         RtlpAcquireSRWLockSharedWait(SRWLock,
 507                                                      &StackWaitBlock,
 508                                                      &SharedWake);
 509
 510                         /* Successfully set the shared count, we acquired the lock */
 511                         break;
 512                     }
 513                 }
 514             }
 515             else
 516             {
 517                 /* This is a fast path, we can simply try to set the shared count to 1 */
 518                 NewValue = (1 << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 519
 520                 /* The RTL_SRWLOCK_CONTENDED bit should never be set if neither the
 521                    RTL_SRWLOCK_SHARED nor the RTL_SRWLOCK_OWNED bit is set */
 522                 ASSERT(!(CurrentValue & RTL_SRWLOCK_CONTENDED));
 523
 524                 if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 525                                                 NewValue,
 526                                                 CurrentValue) == CurrentValue)
 527                 {
 528                     /* Successfully set the shared count, we acquired the lock */
 529                     break;
 530                 }
 531             }
 532         }
 533
 534         YieldProcessor();
 535     }
 536 }
 537
 538
 539 VOID
 540 NTAPI
 541 RtlReleaseSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
 542 {
 543     LONG_PTR CurrentValue, NewValue;
 544     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 545     BOOLEAN LastShared;
 546
 547     while (1)
 548     {
 549         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 550
 551         if (CurrentValue & RTL_SRWLOCK_SHARED)
 552         {
 553             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 554             {
 555                 /* There's a wait block, we need to wake a pending
 556                    exclusive acquirer if this is the last shared release */
 557                 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
 558                 if (WaitBlock != NULL)
 559                 {
 560                     LastShared = (--WaitBlock->SharedCount == 0);
 561
 562                     if (LastShared)
 563                         RtlpReleaseWaitBlockLockLastShared(SRWLock,
 564                                                            WaitBlock);
 565                     else
 566                         RtlpReleaseWaitBlockLock(SRWLock);
 567
 568                     /* We released the lock */
 569                     break;
 570                 }
 571             }
 572             else
 573             {
 574                 /* This is a fast path, we can simply decrement the shared
 575                    count and store the pointer */
 576                 NewValue = CurrentValue >> RTL_SRWLOCK_BITS;
 577
 578                 if (--NewValue != 0)
 579                 {
 580                     NewValue = (NewValue << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 581                 }
 582
 583                 if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 584                                                 NewValue,
 585                                                 CurrentValue) == CurrentValue)
 586                 {
 587                     /* Successfully released the lock */
 588                     break;
 589                 }
 590             }
 591         }
 592         else
 593         {
 594             /* The RTL_SRWLOCK_SHARED bit has to be present now,
 595                even in the contended case! */
 596             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 597         }
 598
 599         YieldProcessor();
 600     }
 601 }
 602
 603
 604 VOID
 605 NTAPI
 606 RtlAcquireSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
 607 {
 608     __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
 609     PRTLP_SRWLOCK_WAITBLOCK First, Last;
 610
 611     if (InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
 612                                         RTL_SRWLOCK_OWNED_BIT))
 613     {
 614         LONG_PTR CurrentValue, NewValue;
 615
 616         while (1)
 617         {
 618             CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 619
 620             if (CurrentValue & RTL_SRWLOCK_SHARED)
 621             {
 622                 /* A shared lock is being held right now. We need to add a wait block! */
 623
 624                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 625                 {
 626                     goto AddWaitBlock;
 627                 }
 628                 else
 629                 {
 630                     /* There are no wait blocks so far, we need to add ourselves as the first
 631                        wait block. We need to keep the shared count! */
 632                     StackWaitBlock.Exclusive = TRUE;
 633                     StackWaitBlock.SharedCount = CurrentValue >> RTL_SRWLOCK_BITS;
 634                     StackWaitBlock.Next = NULL;
 635                     StackWaitBlock.Last = &StackWaitBlock;
 636                     StackWaitBlock.Wake = 0;
 637
 638                     ASSERT_SRW_WAITBLOCK(&StackWaitBlock);
 639
 640                     NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENDED | RTL_SRWLOCK_OWNED;
 641
 642                     if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 643                                                     NewValue,
 644                                                     CurrentValue) == CurrentValue)
 645                     {
 646                         RtlpAcquireSRWLockExclusiveWait(SRWLock,
 647                                                         &StackWaitBlock);
 648
 649                         /* Successfully acquired the exclusive lock */
 650                         break;
 651                     }
 652                 }
 653             }
 654             else
 655             {
 656                 if (CurrentValue & RTL_SRWLOCK_OWNED)
 657                 {
 658                     /* An exclusive lock is being held right now. We need to add a wait block! */
 659
 660                     if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 661                     {
 662 AddWaitBlock:
 663                         StackWaitBlock.Exclusive = TRUE;
 664                         StackWaitBlock.SharedCount = 0;
 665                         StackWaitBlock.Next = NULL;
 666                         StackWaitBlock.Last = &StackWaitBlock;
 667                         StackWaitBlock.Wake = 0;
 668
 669                         ASSERT_SRW_WAITBLOCK(&StackWaitBlock);
 670
 671                         First = RtlpAcquireWaitBlockLock(SRWLock);
 672                         if (First != NULL)
 673                         {
 674                             Last = First->Last;
 675                             Last->Next = &StackWaitBlock;
 676                             First->Last = &StackWaitBlock;
 677
 678                             RtlpReleaseWaitBlockLock(SRWLock);
 679
 680                             RtlpAcquireSRWLockExclusiveWait(SRWLock,
 681                                                             &StackWaitBlock);
 682
 683                             /* Successfully acquired the exclusive lock */
 684                             break;
 685                         }
 686                     }
 687                     else
 688                     {
 689                         /* There are no wait blocks so far, we need to add ourselves as the first
 690                            wait block. We need to keep the shared count! */
 691                         StackWaitBlock.Exclusive = TRUE;
 692                         StackWaitBlock.SharedCount = 0;
 693                         StackWaitBlock.Next = NULL;
 694                         StackWaitBlock.Last = &StackWaitBlock;
 695                         StackWaitBlock.Wake = 0;
 696
 697                         ASSERT_SRW_WAITBLOCK(&StackWaitBlock);
 698
 699                         NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 700                         if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 701                                                         NewValue,
 702                                                         CurrentValue) == CurrentValue)
 703                         {
 704                             RtlpAcquireSRWLockExclusiveWait(SRWLock,
 705                                                             &StackWaitBlock);
 706
 707                             /* Successfully acquired the exclusive lock */
 708                             break;
 709                         }
 710                     }
 711                 }
 712                 else
 713                 {
 714                     if (!InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
 715                                                          RTL_SRWLOCK_OWNED_BIT))
 716                     {
 717                         /* We managed to get hold of a simple exclusive lock! */
 718                         break;
 719                     }
 720                 }
 721             }
 722
 723             YieldProcessor();
 724         }
 725     }
 726 }
 727
 728
 729 VOID
 730 NTAPI
 731 RtlReleaseSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
 732 {
 733     LONG_PTR CurrentValue, NewValue;
 734     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 735
 736     while (1)
 737     {
 738         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 739
 740         if (!(CurrentValue & RTL_SRWLOCK_OWNED))
 741         {
 742             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 743         }
 744
 745         if (!(CurrentValue & RTL_SRWLOCK_SHARED))
 746         {
 747             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 748             {
 749                 /* There's a wait block, we need to wake the next pending
 750                    acquirer (exclusive or shared) */
 751                 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
 752                 if (WaitBlock != NULL)
 753                 {
 754                     RtlpReleaseWaitBlockLockExclusive(SRWLock,
 755                                                       WaitBlock);
 756
 757                     /* We released the lock */
 758                     break;
 759                 }
 760             }
 761             else
 762             {
 763                 /* This is a fast path, we can simply clear the RTL_SRWLOCK_OWNED
 764                    bit. All other bits should be 0 now because this is a simple
 765                    exclusive lock and no one is waiting. */
 766
 767                 ASSERT(!(CurrentValue & ~RTL_SRWLOCK_OWNED));
 768
 769                 NewValue = 0;
 770                 if (_InterlockedCompareExchange((PLONG)&SRWLock->Ptr,
 771                                                 NewValue,
 772                                                 CurrentValue) == CurrentValue)
 773                 {
 774                     /* We released the lock */
 775                     break;
 776                 }
 777             }
 778         }
 779         else
 780         {
 781             /* The RTL_SRWLOCK_SHARED bit must not be present now,
 782                not even in the contended case! */
 783             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 784         }
 785
 786         YieldProcessor();
 787     }
 788 }