reactos/dll/win32/ntdll_vista/srw.c

   1 /*
   2  * COPYRIGHT:         See COPYING in the top level directory
   3  * PROJECT:           ReactOS system libraries
   4  * PURPOSE:           Slim Reader/Writer (SRW) Routines
   5  * PROGRAMMER:        Thomas Weidenmueller <w3seek@reactos.com>
   6  *
   7  * NOTES:             The algorithms used in this implementation
   8  *                    may be different from Vista's implementation.
   9  *                    Since applications should treat the RTL_SRWLOCK
  10  *                    structure as opaque data, it should not matter.
  11  *                    The algorithms are probably not as optimized.
  12  */
  13
  14 /* INCLUDES *****************************************************************/
  15
  16 #include <rtl_vista.h>
  17
  18 #define NDEBUG
  19 #include <debug.h>
  20
  21 /* FUNCTIONS *****************************************************************/
  22
  23 #ifdef _WIN64
  24 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet64((PLONGLONG)ptr,(LONGLONG)val)
  25 #define InterlockedAddPointer(ptr,val) InterlockedAdd64((PLONGLONG)ptr,(LONGLONG)val)
  26 #define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)
  27 #define InterlockedOrPointer(ptr,val) InterlockedOr64((PLONGLONG)ptr,(LONGLONG)val)
  28 #else
  29 #define InterlockedBitTestAndSetPointer(ptr,val) InterlockedBitTestAndSet((PLONG)ptr,(LONG)val)
  30 #define InterlockedAddPointer(ptr,val) InterlockedAdd((PLONG)ptr,(LONG)val)
  31 #define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)
  32 #define InterlockedOrPointer(ptr,val) InterlockedOr((PLONG)ptr,(LONG)val)
  33 #endif
  34
  35 #define RTL_SRWLOCK_OWNED_BIT   0
  36 #define RTL_SRWLOCK_CONTENDED_BIT   1
  37 #define RTL_SRWLOCK_SHARED_BIT  2
  38 #define RTL_SRWLOCK_CONTENTION_LOCK_BIT 3
  39 #define RTL_SRWLOCK_OWNED   (1 << RTL_SRWLOCK_OWNED_BIT)
  40 #define RTL_SRWLOCK_CONTENDED   (1 << RTL_SRWLOCK_CONTENDED_BIT)
  41 #define RTL_SRWLOCK_SHARED  (1 << RTL_SRWLOCK_SHARED_BIT)
  42 #define RTL_SRWLOCK_CONTENTION_LOCK (1 << RTL_SRWLOCK_CONTENTION_LOCK_BIT)
  43 #define RTL_SRWLOCK_MASK    (RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED | \
  44                              RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENTION_LOCK)
  45 #define RTL_SRWLOCK_BITS    4
  46
  47 typedef struct _RTLP_SRWLOCK_SHARED_WAKE
  48 {
  49     LONG Wake;
  50     volatile struct _RTLP_SRWLOCK_SHARED_WAKE *Next;
  51 } volatile RTLP_SRWLOCK_SHARED_WAKE, *PRTLP_SRWLOCK_SHARED_WAKE;
  52
  53 typedef struct _RTLP_SRWLOCK_WAITBLOCK
  54 {
  55     /* SharedCount is the number of shared acquirers. */
  56     LONG SharedCount;
  57
  58     /* Last points to the last wait block in the chain. The value
  59        is only valid when read from the first wait block. */
  60     volatile struct _RTLP_SRWLOCK_WAITBLOCK *Last;
  61
  62     /* Next points to the next wait block in the chain. */
  63     volatile struct _RTLP_SRWLOCK_WAITBLOCK *Next;
  64
  65     union
  66     {
  67         /* Wake is only valid for exclusive wait blocks */
  68         LONG Wake;
  69         /* The wake chain is only valid for shared wait blocks */
  70         struct
  71         {
  72             PRTLP_SRWLOCK_SHARED_WAKE SharedWakeChain;
  73             PRTLP_SRWLOCK_SHARED_WAKE LastSharedWake;
  74         };
  75     };
  76
  77     BOOLEAN Exclusive;
  78 } volatile RTLP_SRWLOCK_WAITBLOCK, *PRTLP_SRWLOCK_WAITBLOCK;
  79
  80
  81 static VOID
  82 NTAPI
  83 RtlpReleaseWaitBlockLockExclusive(IN OUT PRTL_SRWLOCK SRWLock,
  84                                   IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
  85 {
  86     PRTLP_SRWLOCK_WAITBLOCK Next;
  87     LONG_PTR NewValue;
  88
  89     /* NOTE: We're currently in an exclusive lock in contended mode. */
  90
  91     Next = FirstWaitBlock->Next;
  92     if (Next != NULL)
  93     {
  94         /* There's more blocks chained, we need to update the pointers
  95            in the next wait block and update the wait block pointer. */
  96         NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
  97         if (!FirstWaitBlock->Exclusive)
  98         {
  99             /* The next wait block has to be an exclusive lock! */
 100             ASSERT(Next->Exclusive);
 101
 102             /* Save the shared count */
 103             Next->SharedCount = FirstWaitBlock->SharedCount;
 104
 105             NewValue |= RTL_SRWLOCK_SHARED;
 106         }
 107
 108         Next->Last = FirstWaitBlock->Last;
 109     }
 110     else
 111     {
 112         /* Convert the lock to a simple lock. */
 113         if (FirstWaitBlock->Exclusive)
 114             NewValue = RTL_SRWLOCK_OWNED;
 115         else
 116         {
 117             ASSERT(FirstWaitBlock->SharedCount > 0);
 118
 119             NewValue = ((LONG_PTR)FirstWaitBlock->SharedCount << RTL_SRWLOCK_BITS) |
 120                        RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 121         }
 122     }
 123
 124     (void)InterlockedExchangePointer(&SRWLock->Ptr, (PVOID)NewValue);
 125
 126     if (FirstWaitBlock->Exclusive)
 127     {
 128         (void)InterlockedOr(&FirstWaitBlock->Wake,
 129                             TRUE);
 130     }
 131     else
 132     {
 133         PRTLP_SRWLOCK_SHARED_WAKE WakeChain, NextWake;
 134
 135         /* If we were the first one to acquire the shared
 136            lock, we now need to wake all others... */
 137         WakeChain = FirstWaitBlock->SharedWakeChain;
 138         do
 139         {
 140             NextWake = WakeChain->Next;
 141
 142             (void)InterlockedOr((PLONG)&WakeChain->Wake,
 143                                 TRUE);
 144
 145             WakeChain = NextWake;
 146         } while (WakeChain != NULL);
 147     }
 148 }
 149
 150
 151 static VOID
 152 NTAPI
 153 RtlpReleaseWaitBlockLockLastShared(IN OUT PRTL_SRWLOCK SRWLock,
 154                                    IN PRTLP_SRWLOCK_WAITBLOCK FirstWaitBlock)
 155 {
 156     PRTLP_SRWLOCK_WAITBLOCK Next;
 157     LONG_PTR NewValue;
 158
 159     /* NOTE: We're currently in a shared lock in contended mode. */
 160
 161     /* The next acquirer to be unwaited *must* be an exclusive lock! */
 162     ASSERT(FirstWaitBlock->Exclusive);
 163
 164     Next = FirstWaitBlock->Next;
 165     if (Next != NULL)
 166     {
 167         /* There's more blocks chained, we need to update the pointers
 168            in the next wait block and update the wait block pointer. */
 169         NewValue = (LONG_PTR)Next | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 170
 171         Next->Last = FirstWaitBlock->Last;
 172     }
 173     else
 174     {
 175         /* Convert the lock to a simple exclusive lock. */
 176         NewValue = RTL_SRWLOCK_OWNED;
 177     }
 178
 179     (void)InterlockedExchangePointer(&SRWLock->Ptr, (PVOID)NewValue);
 180
 181     (void)InterlockedOr(&FirstWaitBlock->Wake,
 182                         TRUE);
 183 }
 184
 185
 186 static VOID
 187 NTAPI
 188 RtlpReleaseWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
 189 {
 190     InterlockedAndPointer(&SRWLock->Ptr,
 191                           ~RTL_SRWLOCK_CONTENTION_LOCK);
 192 }
 193
 194
 195 static PRTLP_SRWLOCK_WAITBLOCK
 196 NTAPI
 197 RtlpAcquireWaitBlockLock(IN OUT PRTL_SRWLOCK SRWLock)
 198 {
 199     LONG_PTR PrevValue;
 200     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 201
 202     while (1)
 203     {
 204         PrevValue = InterlockedOrPointer(&SRWLock->Ptr,
 205                                          RTL_SRWLOCK_CONTENTION_LOCK);
 206
 207         if (!(PrevValue & RTL_SRWLOCK_CONTENTION_LOCK))
 208             break;
 209
 210         YieldProcessor();
 211     }
 212
 213     if (!(PrevValue & RTL_SRWLOCK_CONTENDED) ||
 214         (PrevValue & ~RTL_SRWLOCK_MASK) == 0)
 215     {
 216         /* Too bad, looks like the wait block was removed in the
 217            meanwhile, unlock again */
 218         RtlpReleaseWaitBlockLock(SRWLock);
 219         return NULL;
 220     }
 221
 222     WaitBlock = (PRTLP_SRWLOCK_WAITBLOCK)(PrevValue & ~RTL_SRWLOCK_MASK);
 223
 224     return WaitBlock;
 225 }
 226
 227
 228 static VOID
 229 NTAPI
 230 RtlpAcquireSRWLockExclusiveWait(IN OUT PRTL_SRWLOCK SRWLock,
 231                                 IN PRTLP_SRWLOCK_WAITBLOCK WaitBlock)
 232 {
 233     LONG_PTR CurrentValue;
 234
 235     while (1)
 236     {
 237         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 238         if (!(CurrentValue & RTL_SRWLOCK_SHARED))
 239         {
 240             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 241             {
 242                 if (WaitBlock->Wake != 0)
 243                 {
 244                     /* Our wait block became the first one
 245                        in the chain, we own the lock now! */
 246                     break;
 247                 }
 248             }
 249             else
 250             {
 251                 /* The last wait block was removed and/or we're
 252                    finally a simple exclusive lock. This means we
 253                    don't need to wait anymore, we acquired the lock! */
 254                 break;
 255             }
 256         }
 257
 258         YieldProcessor();
 259     }
 260 }
 261
 262
 263 static VOID
 264 NTAPI
 265 RtlpAcquireSRWLockSharedWait(IN OUT PRTL_SRWLOCK SRWLock,
 266                              IN OUT PRTLP_SRWLOCK_WAITBLOCK FirstWait  OPTIONAL,
 267                              IN OUT PRTLP_SRWLOCK_SHARED_WAKE WakeChain)
 268 {
 269     if (FirstWait != NULL)
 270     {
 271         while (WakeChain->Wake == 0)
 272         {
 273             YieldProcessor();
 274         }
 275     }
 276     else
 277     {
 278         LONG_PTR CurrentValue;
 279
 280         while (1)
 281         {
 282             CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 283             if (CurrentValue & RTL_SRWLOCK_SHARED)
 284             {
 285                 /* The RTL_SRWLOCK_OWNED bit always needs to be set when
 286                    RTL_SRWLOCK_SHARED is set! */
 287                 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
 288
 289                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 290                 {
 291                     if (WakeChain->Wake != 0)
 292                     {
 293                         /* Our wait block became the first one
 294                            in the chain, we own the lock now! */
 295                         break;
 296                     }
 297                 }
 298                 else
 299                 {
 300                     /* The last wait block was removed and/or we're
 301                        finally a simple shared lock. This means we
 302                        don't need to wait anymore, we acquired the lock! */
 303                     break;
 304                 }
 305             }
 306
 307             YieldProcessor();
 308         }
 309     }
 310 }
 311
 312
 313 VOID
 314 NTAPI
 315 RtlInitializeSRWLock(OUT PRTL_SRWLOCK SRWLock)
 316 {
 317     SRWLock->Ptr = NULL;
 318 }
 319
 320
 321 VOID
 322 NTAPI
 323 RtlAcquireSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
 324 {
 325     __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
 326     RTLP_SRWLOCK_SHARED_WAKE SharedWake;
 327     LONG_PTR CurrentValue, NewValue;
 328     PRTLP_SRWLOCK_WAITBLOCK First, Shared, FirstWait;
 329
 330     while (1)
 331     {
 332         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 333
 334         if (CurrentValue & RTL_SRWLOCK_SHARED)
 335         {
 336             /* NOTE: It is possible that the RTL_SRWLOCK_OWNED bit is set! */
 337
 338             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 339             {
 340                 /* There's other waiters already, lock the wait blocks and
 341                    increment the shared count */
 342                 First = RtlpAcquireWaitBlockLock(SRWLock);
 343                 if (First != NULL)
 344                 {
 345                     FirstWait = NULL;
 346
 347                     if (First->Exclusive)
 348                     {
 349                         /* We need to setup a new wait block! Although
 350                            we're currently in a shared lock and we're acquiring
 351                            a shared lock, there are exclusive locks queued. We need
 352                            to wait until those are released. */
 353                         Shared = First->Last;
 354
 355                         if (Shared->Exclusive)
 356                         {
 357                             StackWaitBlock.Exclusive = FALSE;
 358                             StackWaitBlock.SharedCount = 1;
 359                             StackWaitBlock.Next = NULL;
 360                             StackWaitBlock.Last = &StackWaitBlock;
 361                             StackWaitBlock.SharedWakeChain = &SharedWake;
 362
 363                             Shared->Next = &StackWaitBlock;
 364                             First->Last = &StackWaitBlock;
 365
 366                             Shared = &StackWaitBlock;
 367                             FirstWait = &StackWaitBlock;
 368                         }
 369                         else
 370                         {
 371                             Shared->LastSharedWake->Next = &SharedWake;
 372                             Shared->SharedCount++;
 373                         }
 374                     }
 375                     else
 376                     {
 377                         Shared = First;
 378                         Shared->LastSharedWake->Next = &SharedWake;
 379                         Shared->SharedCount++;
 380                     }
 381
 382                     SharedWake.Next = NULL;
 383                     SharedWake.Wake = 0;
 384
 385                     Shared->LastSharedWake = &SharedWake;
 386
 387                     RtlpReleaseWaitBlockLock(SRWLock);
 388
 389                     RtlpAcquireSRWLockSharedWait(SRWLock,
 390                                                  FirstWait,
 391                                                  &SharedWake);
 392
 393                     /* Successfully incremented the shared count, we acquired the lock */
 394                     break;
 395                 }
 396             }
 397             else
 398             {
 399                 /* This is a fastest path, just increment the number of
 400                    current shared locks */
 401
 402                 /* Since the RTL_SRWLOCK_SHARED bit is set, the RTL_SRWLOCK_OWNED bit also has
 403                    to be set! */
 404
 405                 ASSERT(CurrentValue & RTL_SRWLOCK_OWNED);
 406
 407                 NewValue = (CurrentValue >> RTL_SRWLOCK_BITS) + 1;
 408                 NewValue = (NewValue << RTL_SRWLOCK_BITS) | (CurrentValue & RTL_SRWLOCK_MASK);
 409
 410                 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 411                                                                 (PVOID)NewValue,
 412                                                                 (PVOID)CurrentValue) == CurrentValue)
 413                 {
 414                     /* Successfully incremented the shared count, we acquired the lock */
 415                     break;
 416                 }
 417             }
 418         }
 419         else
 420         {
 421             if (CurrentValue & RTL_SRWLOCK_OWNED)
 422             {
 423                 /* The resource is currently acquired exclusively */
 424                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 425                 {
 426                     SharedWake.Next = NULL;
 427                     SharedWake.Wake = 0;
 428
 429                     /* There's other waiters already, lock the wait blocks and
 430                        increment the shared count. If the last block in the chain
 431                        is an exclusive lock, add another block. */
 432
 433                     StackWaitBlock.Exclusive = FALSE;
 434                     StackWaitBlock.SharedCount = 0;
 435                     StackWaitBlock.Next = NULL;
 436                     StackWaitBlock.Last = &StackWaitBlock;
 437                     StackWaitBlock.SharedWakeChain = &SharedWake;
 438
 439                     First = RtlpAcquireWaitBlockLock(SRWLock);
 440                     if (First != NULL)
 441                     {
 442                         Shared = First->Last;
 443                         if (Shared->Exclusive)
 444                         {
 445                             Shared->Next = &StackWaitBlock;
 446                             First->Last = &StackWaitBlock;
 447
 448                             Shared = &StackWaitBlock;
 449                             FirstWait = &StackWaitBlock;
 450                         }
 451                         else
 452                         {
 453                             FirstWait = NULL;
 454                             Shared->LastSharedWake->Next = &SharedWake;
 455                         }
 456
 457                         Shared->SharedCount++;
 458                         Shared->LastSharedWake = &SharedWake;
 459
 460                         RtlpReleaseWaitBlockLock(SRWLock);
 461
 462                         RtlpAcquireSRWLockSharedWait(SRWLock,
 463                                                      FirstWait,
 464                                                      &SharedWake);
 465
 466                         /* Successfully incremented the shared count, we acquired the lock */
 467                         break;
 468                     }
 469                 }
 470                 else
 471                 {
 472                     SharedWake.Next = NULL;
 473                     SharedWake.Wake = 0;
 474
 475                     /* We need to setup the first wait block. Currently an exclusive lock is
 476                        held, change the lock to contended mode. */
 477                     StackWaitBlock.Exclusive = FALSE;
 478                     StackWaitBlock.SharedCount = 1;
 479                     StackWaitBlock.Next = NULL;
 480                     StackWaitBlock.Last = &StackWaitBlock;
 481                     StackWaitBlock.SharedWakeChain = &SharedWake;
 482                     StackWaitBlock.LastSharedWake = &SharedWake;
 483
 484                     NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 485                     if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 486                                                                     (PVOID)NewValue,
 487                                                                     (PVOID)CurrentValue) == CurrentValue)
 488                     {
 489                         RtlpAcquireSRWLockSharedWait(SRWLock,
 490                                                      &StackWaitBlock,
 491                                                      &SharedWake);
 492
 493                         /* Successfully set the shared count, we acquired the lock */
 494                         break;
 495                     }
 496                 }
 497             }
 498             else
 499             {
 500                 /* This is a fast path, we can simply try to set the shared count to 1 */
 501                 NewValue = (1 << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 502
 503                 /* The RTL_SRWLOCK_CONTENDED bit should never be set if neither the
 504                    RTL_SRWLOCK_SHARED nor the RTL_SRWLOCK_OWNED bit is set */
 505                 ASSERT(!(CurrentValue & RTL_SRWLOCK_CONTENDED));
 506
 507                 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 508                                                                 (PVOID)NewValue,
 509                                                                 (PVOID)CurrentValue) == CurrentValue)
 510                 {
 511                     /* Successfully set the shared count, we acquired the lock */
 512                     break;
 513                 }
 514             }
 515         }
 516
 517         YieldProcessor();
 518     }
 519 }
 520
 521
 522 VOID
 523 NTAPI
 524 RtlReleaseSRWLockShared(IN OUT PRTL_SRWLOCK SRWLock)
 525 {
 526     LONG_PTR CurrentValue, NewValue;
 527     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 528     BOOLEAN LastShared;
 529
 530     while (1)
 531     {
 532         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 533
 534         if (CurrentValue & RTL_SRWLOCK_SHARED)
 535         {
 536             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 537             {
 538                 /* There's a wait block, we need to wake a pending
 539                    exclusive acquirer if this is the last shared release */
 540                 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
 541                 if (WaitBlock != NULL)
 542                 {
 543                     LastShared = (--WaitBlock->SharedCount == 0);
 544
 545                     if (LastShared)
 546                         RtlpReleaseWaitBlockLockLastShared(SRWLock,
 547                                                            WaitBlock);
 548                     else
 549                         RtlpReleaseWaitBlockLock(SRWLock);
 550
 551                     /* We released the lock */
 552                     break;
 553                 }
 554             }
 555             else
 556             {
 557                 /* This is a fast path, we can simply decrement the shared
 558                    count and store the pointer */
 559                 NewValue = CurrentValue >> RTL_SRWLOCK_BITS;
 560
 561                 if (--NewValue != 0)
 562                 {
 563                     NewValue = (NewValue << RTL_SRWLOCK_BITS) | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_OWNED;
 564                 }
 565
 566                 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 567                                                                 (PVOID)NewValue,
 568                                                                 (PVOID)CurrentValue) == CurrentValue)
 569                 {
 570                     /* Successfully released the lock */
 571                     break;
 572                 }
 573             }
 574         }
 575         else
 576         {
 577             /* The RTL_SRWLOCK_SHARED bit has to be present now,
 578                even in the contended case! */
 579             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 580         }
 581
 582         YieldProcessor();
 583     }
 584 }
 585
 586
 587 VOID
 588 NTAPI
 589 RtlAcquireSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
 590 {
 591     __ALIGNED(16) RTLP_SRWLOCK_WAITBLOCK StackWaitBlock;
 592     PRTLP_SRWLOCK_WAITBLOCK First, Last;
 593
 594     if (InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
 595                                         RTL_SRWLOCK_OWNED_BIT))
 596     {
 597         LONG_PTR CurrentValue, NewValue;
 598
 599         while (1)
 600         {
 601             CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 602
 603             if (CurrentValue & RTL_SRWLOCK_SHARED)
 604             {
 605                 /* A shared lock is being held right now. We need to add a wait block! */
 606
 607                 if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 608                 {
 609                     goto AddWaitBlock;
 610                 }
 611                 else
 612                 {
 613                     /* There are no wait blocks so far, we need to add ourselves as the first
 614                        wait block. We need to keep the shared count! */
 615                     StackWaitBlock.Exclusive = TRUE;
 616                     StackWaitBlock.SharedCount = (LONG)(CurrentValue >> RTL_SRWLOCK_BITS);
 617                     StackWaitBlock.Next = NULL;
 618                     StackWaitBlock.Last = &StackWaitBlock;
 619                     StackWaitBlock.Wake = 0;
 620
 621                     NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_SHARED | RTL_SRWLOCK_CONTENDED | RTL_SRWLOCK_OWNED;
 622
 623                     if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 624                                                                     (PVOID)NewValue,
 625                                                                     (PVOID)CurrentValue) == CurrentValue)
 626                     {
 627                         RtlpAcquireSRWLockExclusiveWait(SRWLock,
 628                                                         &StackWaitBlock);
 629
 630                         /* Successfully acquired the exclusive lock */
 631                         break;
 632                     }
 633                 }
 634             }
 635             else
 636             {
 637                 if (CurrentValue & RTL_SRWLOCK_OWNED)
 638                 {
 639                     /* An exclusive lock is being held right now. We need to add a wait block! */
 640
 641                     if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 642                     {
 643 AddWaitBlock:
 644                         StackWaitBlock.Exclusive = TRUE;
 645                         StackWaitBlock.SharedCount = 0;
 646                         StackWaitBlock.Next = NULL;
 647                         StackWaitBlock.Last = &StackWaitBlock;
 648                         StackWaitBlock.Wake = 0;
 649
 650                         First = RtlpAcquireWaitBlockLock(SRWLock);
 651                         if (First != NULL)
 652                         {
 653                             Last = First->Last;
 654                             Last->Next = &StackWaitBlock;
 655                             First->Last = &StackWaitBlock;
 656
 657                             RtlpReleaseWaitBlockLock(SRWLock);
 658
 659                             RtlpAcquireSRWLockExclusiveWait(SRWLock,
 660                                                             &StackWaitBlock);
 661
 662                             /* Successfully acquired the exclusive lock */
 663                             break;
 664                         }
 665                     }
 666                     else
 667                     {
 668                         /* There are no wait blocks so far, we need to add ourselves as the first
 669                            wait block. We need to keep the shared count! */
 670                         StackWaitBlock.Exclusive = TRUE;
 671                         StackWaitBlock.SharedCount = 0;
 672                         StackWaitBlock.Next = NULL;
 673                         StackWaitBlock.Last = &StackWaitBlock;
 674                         StackWaitBlock.Wake = 0;
 675
 676                         NewValue = (ULONG_PTR)&StackWaitBlock | RTL_SRWLOCK_OWNED | RTL_SRWLOCK_CONTENDED;
 677                         if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 678                                                                         (PVOID)NewValue,
 679                                                                         (PVOID)CurrentValue) == CurrentValue)
 680                         {
 681                             RtlpAcquireSRWLockExclusiveWait(SRWLock,
 682                                                             &StackWaitBlock);
 683
 684                             /* Successfully acquired the exclusive lock */
 685                             break;
 686                         }
 687                     }
 688                 }
 689                 else
 690                 {
 691                     if (!InterlockedBitTestAndSetPointer(&SRWLock->Ptr,
 692                                                          RTL_SRWLOCK_OWNED_BIT))
 693                     {
 694                         /* We managed to get hold of a simple exclusive lock! */
 695                         break;
 696                     }
 697                 }
 698             }
 699
 700             YieldProcessor();
 701         }
 702     }
 703 }
 704
 705
 706 VOID
 707 NTAPI
 708 RtlReleaseSRWLockExclusive(IN OUT PRTL_SRWLOCK SRWLock)
 709 {
 710     LONG_PTR CurrentValue, NewValue;
 711     PRTLP_SRWLOCK_WAITBLOCK WaitBlock;
 712
 713     while (1)
 714     {
 715         CurrentValue = *(volatile LONG_PTR *)&SRWLock->Ptr;
 716
 717         if (!(CurrentValue & RTL_SRWLOCK_OWNED))
 718         {
 719             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 720         }
 721
 722         if (!(CurrentValue & RTL_SRWLOCK_SHARED))
 723         {
 724             if (CurrentValue & RTL_SRWLOCK_CONTENDED)
 725             {
 726                 /* There's a wait block, we need to wake the next pending
 727                    acquirer (exclusive or shared) */
 728                 WaitBlock = RtlpAcquireWaitBlockLock(SRWLock);
 729                 if (WaitBlock != NULL)
 730                 {
 731                     RtlpReleaseWaitBlockLockExclusive(SRWLock,
 732                                                       WaitBlock);
 733
 734                     /* We released the lock */
 735                     break;
 736                 }
 737             }
 738             else
 739             {
 740                 /* This is a fast path, we can simply clear the RTL_SRWLOCK_OWNED
 741                    bit. All other bits should be 0 now because this is a simple
 742                    exclusive lock and no one is waiting. */
 743
 744                 ASSERT(!(CurrentValue & ~RTL_SRWLOCK_OWNED));
 745
 746                 NewValue = 0;
 747                 if ((LONG_PTR)InterlockedCompareExchangePointer(&SRWLock->Ptr,
 748                                                                 (PVOID)NewValue,
 749                                                                 (PVOID)CurrentValue) == CurrentValue)
 750                 {
 751                     /* We released the lock */
 752                     break;
 753                 }
 754             }
 755         }
 756         else
 757         {
 758             /* The RTL_SRWLOCK_SHARED bit must not be present now,
 759                not even in the contended case! */
 760             RtlRaiseStatus(STATUS_RESOURCE_NOT_OWNED);
 761         }
 762
 763         YieldProcessor();
 764     }
 765 }