2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/pushlock.c
5 * PURPOSE: Pushlock and Cache-Aware Pushlock Implementation
6 * PROGRAMMER: Alex Ionescu (alex.ionescu@reactos.com)
9 /* INCLUDES *****************************************************************/
13 #include <internal/debug.h>
15 /* DATA **********************************************************************/
17 ULONG ExPushLockSpinCount
;
19 /* PRIVATE FUNCTIONS *********************************************************/
22 * @name ExpInitializePushLocks
24 * The ExpInitializePushLocks routine initialized Pushlock support.
30 * @remarks The ExpInitializePushLocks routine sets up the spin on SMP machines.
35 ExpInitializePushLocks(VOID
)
37 /* Initialize an internal 1024-iteration spin for MP CPUs */
38 ExPushLockSpinCount
= (KeNumberProcessors
== 1) ? 0 : 1024;
42 * @name ExfWakePushLock
44 * The ExfWakePushLock routine wakes a Pushlock that is in the waiting
48 * Pointer to a pushlock that is waiting.
51 * Last known value of the pushlock before this routine was called.
55 * @remarks This is an internal routine; do not call it manually. Only the system
56 * can properly know if the pushlock is ready to be awakened or not.
57 * External callers should use ExfTrytoWakePushLock.
62 ExfWakePushLock(PEX_PUSH_LOCK PushLock
,
63 EX_PUSH_LOCK OldValue
)
65 EX_PUSH_LOCK NewValue
;
66 PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock
, FirstWaitBlock
, NextWaitBlock
;
67 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
;
70 /* Start main wake loop */
74 ASSERT(!OldValue
.MultipleShared
);
76 /* Check if it's locked */
79 /* If it's locked we must simply un-wake it*/
82 /* It's not waking anymore */
83 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_WAKING
;
86 ASSERT(!NewValue
.Waking
);
87 ASSERT(NewValue
.Locked
);
88 ASSERT(NewValue
.Waiting
);
90 /* Write the New Value */
91 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
94 if (NewValue
.Value
== OldValue
.Value
) return;
96 /* Someone changed the value behind our back, update it*/
99 /* Check if it's still locked */
100 if (!OldValue
.Locked
) break;
104 /* Save the First Block */
105 FirstWaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)OldValue
.Ptr
&
106 ~EX_PUSH_LOCK_PTR_BITS
);
107 NextWaitBlock
= FirstWaitBlock
;
108 WaitBlock
= NextWaitBlock
->Last
;
110 /* Try to find a wait block */
113 /* Save the previous block */
114 PreviousWaitBlock
= NextWaitBlock
;
116 /* Move to next block */
117 NextWaitBlock
= NextWaitBlock
->Next
;
119 /* Save the previous block */
120 NextWaitBlock
->Previous
= PreviousWaitBlock
;
122 /* Move to the next one */
123 WaitBlock
= NextWaitBlock
->Last
;
126 /* Check if the last Wait Block is not Exclusive or if it's the only one */
127 PreviousWaitBlock
= WaitBlock
->Previous
;
128 if (!(WaitBlock
->Flags
& EX_PUSH_LOCK_FLAGS_EXCLUSIVE
) ||
129 !(PreviousWaitBlock
))
131 /* Destroy the pushlock */
132 if (InterlockedCompareExchangePointer(PushLock
, 0, OldValue
.Ptr
) ==
137 /* Link the wait blocks */
138 FirstWaitBlock
->Last
= PreviousWaitBlock
;
139 WaitBlock
->Previous
= NULL
;
142 ASSERT(FirstWaitBlock
!= WaitBlock
);
143 ASSERT(PushLock
->Waiting
);
145 /* Remove waking bit from pushlock */
146 InterlockedAnd((PLONG
)PushLock
, ~EX_PUSH_LOCK_WAKING
);
150 /* Check if there's a previous block */
151 OldIrql
= DISPATCH_LEVEL
;
152 if (WaitBlock
->Previous
)
154 /* Raise to Dispatch */
155 KeRaiseIrql(DISPATCH_LEVEL
, &OldIrql
);
161 /* Get the previous Wait block */
162 PreviousWaitBlock
= WaitBlock
->Previous
;
165 ASSERT(!WaitBlock
->Signaled
);
168 /* We are about to get signaled */
169 WaitBlock
->Signaled
= TRUE
;
172 /* Set the Wait Bit in the Wait Block */
173 if (!InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
175 /* Nobody signaled us, so do it */
176 KeSignalGateBoostPriority(&WaitBlock
->WakeGate
);
179 /* Set the wait block and check if there still is one to loop*/
180 WaitBlock
= PreviousWaitBlock
;
181 if (!WaitBlock
) break;
184 /* Check if we have to lower back the IRQL */
185 if (OldIrql
!= DISPATCH_LEVEL
) KeLowerIrql(OldIrql
);
189 * @name ExpOptimizePushLockList
191 * The ExpOptimizePushLockList routine optimizes the list of waiters
192 * associated to a pushlock's wait block.
195 * Pointer to a pushlock whose waiter list needs to be optimized.
198 * Last known value of the pushlock before this routine was called.
202 * @remarks At the end of the optimization, the pushlock will also be wakened.
207 ExpOptimizePushLockList(PEX_PUSH_LOCK PushLock
,
208 EX_PUSH_LOCK OldValue
)
210 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, LastWaitBlock
, PreviousWaitBlock
;
211 EX_PUSH_LOCK NewValue
;
213 /* Check if the pushlock is locked */
216 /* Start main loop */
219 /* Get the wait block */
220 WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)OldValue
.Ptr
&
221 ~EX_PUSH_LOCK_PTR_BITS
);
223 /* Loop the blocks */
224 LastWaitBlock
= WaitBlock
->Last
;
225 while (LastWaitBlock
)
228 PreviousWaitBlock
= WaitBlock
;
230 /* Get the next block */
231 WaitBlock
= WaitBlock
->Next
;
233 /* Save the previous */
234 WaitBlock
->Previous
= PreviousWaitBlock
;
236 /* Move to the next */
237 LastWaitBlock
= WaitBlock
->Last
;
240 /* Remove the wake bit */
241 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_WAKING
;
244 ASSERT(NewValue
.Locked
);
245 ASSERT(!NewValue
.Waking
);
247 /* Update the value */
248 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
252 /* If we updated correctly, leave */
253 if (NewValue
.Value
== OldValue
.Value
) return;
255 /* If the value is now locked, loop again */
256 if (NewValue
.Locked
) continue;
260 /* Wake the push lock */
261 ExfWakePushLock(PushLock
, OldValue
);
265 * @name ExTimedWaitForUnblockPushLock
267 * The ExTimedWaitForUnblockPushLock routine waits for a pushlock
268 * to be unblocked, for a specified internal.
271 * Pointer to a pushlock whose waiter list needs to be optimized.
274 * Pointer to the pushlock's wait block.
277 * Amount of time to wait for this pushlock to be unblocked.
279 * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
280 * code returned by KeWaitForSingleObject.
282 * @remarks If the wait fails, then a manual unblock is attempted.
287 ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock
,
289 IN PLARGE_INTEGER Timeout
)
294 /* Initialize the wait event */
295 KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->WakeEvent
,
299 /* Spin on the push lock if necessary */
300 i
= ExPushLockSpinCount
;
306 /* Check if we got lucky and can leave early */
307 if (!(((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->Flags
&
308 EX_PUSH_LOCK_WAITING
))
310 /* This wait block isn't waiting anymore, we can leave */
311 return STATUS_SUCCESS
;
317 /* Now try to remove the wait bit */
318 if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->Flags
,
321 /* Nobody removed it already, let's do a full wait */
322 Status
= KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->
328 if (!NT_SUCCESS(Status
))
330 /* Try unblocking the pushlock */
331 ExfUnblockPushLock(PushLock
, WaitBlock
);
336 /* Someone beat us to it, no need to wait */
337 Status
= STATUS_SUCCESS
;
345 * @name ExWaitForUnblockPushLock
347 * The ExWaitForUnblockPushLock routine waits for a pushlock
348 * to be unblocked, for a specified internal.
351 * Pointer to a pushlock whose waiter list needs to be optimized.
354 * Pointer to the pushlock's wait block.
356 * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
357 * code returned by KeWaitForSingleObject.
359 * @remarks If the wait fails, then a manual unblock is attempted.
364 ExWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock
,
367 /* Call the timed function with no timeout */
368 ExTimedWaitForUnblockPushLock(PushLock
, WaitBlock
, NULL
);
372 * @name ExBlockPushLock
374 * The ExBlockPushLock routine blocks a pushlock.
377 * Pointer to a pushlock whose waiter list needs to be optimized.
380 * Pointer to the pushlock's wait block.
389 ExBlockPushLock(PEX_PUSH_LOCK PushLock
,
392 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
= pWaitBlock
;
393 EX_PUSH_LOCK NewValue
, OldValue
;
395 /* Detect invalid wait block alignment */
396 ASSERT((ULONG_PTR
)pWaitBlock
& 0x10);
398 /* Set the waiting bit */
399 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_WAIT
;
401 /* Get the old value */
402 OldValue
= *PushLock
;
404 /* Start block loop */
407 /* Link the wait blocks */
408 WaitBlock
->Next
= OldValue
.Ptr
;
410 /* Set the new wait block value */
411 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
414 if (OldValue
.Ptr
== NewValue
.Ptr
) break;
416 /* Try again with the new value */
421 /* PUBLIC FUNCTIONS **********************************************************/
424 * @name ExAcquirePushLockExclusive
427 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
430 * Pointer to the pushlock which is to be acquired.
434 * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
435 * This macro should usually be paired up with KeAcquireCriticalRegion.
440 ExfAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
442 DEFINE_WAIT_BLOCK(WaitBlock
);
443 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
, TempValue
;
447 /* Start main loop */
450 /* Check if it's unlocked */
451 if (!OldValue
.Locked
)
454 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
455 ASSERT(NewValue
.Locked
);
457 /* Set the new value */
458 if (InterlockedCompareExchangePointer(PushLock
,
460 OldValue
.Ptr
) != OldValue
.Ptr
)
463 OldValue
= *PushLock
;
467 /* Break out of the loop */
472 /* We'll have to create a Waitblock */
473 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_EXCLUSIVE
|
474 EX_PUSH_LOCK_FLAGS_WAIT
;
475 WaitBlock
->Previous
= NULL
;
478 /* Check if there is already a waiter */
479 if (OldValue
.Waiting
)
481 /* Nobody is the last waiter yet */
482 WaitBlock
->Last
= NULL
;
484 /* We are an exclusive waiter */
485 WaitBlock
->ShareCount
= 0;
487 /* Set the current Wait Block pointer */
488 WaitBlock
->Next
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)
489 OldValue
.Ptr
&~ EX_PUSH_LOCK_PTR_BITS
);
492 NewValue
.Value
= (OldValue
.Value
& EX_PUSH_LOCK_MULTIPLE_SHARED
) |
494 EX_PUSH_LOCK_WAKING
|
495 EX_PUSH_LOCK_WAITING
|
496 PtrToUlong(&WaitBlock
);
498 /* Check if the pushlock was already waking */
499 if (OldValue
.Waking
) NeedWake
= TRUE
;
503 /* We are the first waiter, so loop the wait block */
504 WaitBlock
->Last
= WaitBlock
;
506 /* Set the share count */
507 WaitBlock
->ShareCount
= OldValue
.Shared
;
509 /* Check if someone is sharing this pushlock */
510 if (OldValue
.Shared
> 1)
512 /* Point to our wait block */
513 NewValue
.Value
= EX_PUSH_LOCK_MULTIPLE_SHARED
|
515 EX_PUSH_LOCK_WAITING
|
516 PtrToUlong(&WaitBlock
);
520 /* No shared count */
521 WaitBlock
->ShareCount
= 0;
523 /* Point to our wait block */
524 NewValue
.Value
= EX_PUSH_LOCK_LOCK
|
525 EX_PUSH_LOCK_WAITING
|
526 PtrToUlong(&WaitBlock
);
531 /* Setup the Debug Wait Block */
532 WaitBlock
->Signaled
= 0;
533 WaitBlock
->OldValue
= OldValue
;
534 WaitBlock
->NewValue
= NewValue
;
535 WaitBlock
->PushLock
= PushLock
;
539 ASSERT(NewValue
.Waiting
);
540 ASSERT(NewValue
.Locked
);
542 /* Write the new value */
543 TempValue
= NewValue
;
544 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
547 if (NewValue
.Value
!= OldValue
.Value
)
550 OldValue
= *PushLock
;
554 /* Check if the pushlock needed waking */
557 /* Scan the Waiters and Wake PushLocks */
558 ExpOptimizePushLockList(PushLock
, TempValue
);
561 /* Set up the Wait Gate */
562 KeInitializeGate(&WaitBlock
->WakeGate
);
564 /* Now spin on the push lock if necessary */
565 i
= ExPushLockSpinCount
;
566 if ((i
) && (WaitBlock
->Flags
& EX_PUSH_LOCK_WAITING
))
569 while (--i
) YieldProcessor();
572 /* Now try to remove the wait bit */
573 if (InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
575 /* Nobody removed it already, let's do a full wait */
576 KeWaitForGate(&WaitBlock
->WakeGate
, WrPushLock
, KernelMode
);
577 ASSERT(WaitBlock
->Signaled
);
580 /* We shouldn't be shared anymore */
581 ASSERT((WaitBlock
->ShareCount
== 0));
590 * @name ExAcquirePushLockExclusive
593 * The ExAcquirePushLockShared macro acquires a shared PushLock.
596 * Pointer to the pushlock which is to be acquired.
600 * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
601 * This macro should usually be paired up with KeAcquireCriticalRegion.
606 ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
608 DEFINE_WAIT_BLOCK(WaitBlock
);
609 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
;
613 /* Start main loop */
616 /* Check if it's unlocked or if it's waiting without any sharers */
617 if (!(OldValue
.Locked
) || (OldValue
.Waiting
&& OldValue
.Shared
== 0))
619 /* Check if anyone is waiting on it */
620 if (!OldValue
.Waiting
)
622 /* Increase the share count and lock it */
623 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
628 /* Simply set the lock bit */
629 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
633 ASSERT(NewValue
.Locked
);
635 /* Set the new value */
636 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
639 if (NewValue
.Value
!= OldValue
.Value
)
646 /* Break out of the loop */
651 /* We'll have to create a Waitblock */
652 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_WAIT
;
653 WaitBlock
->ShareCount
= 0;
655 WaitBlock
->Previous
= NULL
;
657 /* Check if there is already a waiter */
658 if (OldValue
.Waiting
)
660 /* Set the current Wait Block pointer */
661 WaitBlock
->Next
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)
662 OldValue
.Ptr
&~ EX_PUSH_LOCK_PTR_BITS
);
664 /* Nobody is the last waiter yet */
665 WaitBlock
->Last
= NULL
;
668 NewValue
.Value
= (OldValue
.Value
& (EX_PUSH_LOCK_MULTIPLE_SHARED
|
669 EX_PUSH_LOCK_LOCK
)) |
670 EX_PUSH_LOCK_WAKING
|
671 EX_PUSH_LOCK_WAITING
|
672 PtrToUlong(&WaitBlock
);
674 /* Check if the pushlock was already waking */
675 if (OldValue
.Waking
) NeedWake
= TRUE
;
679 /* We are the first waiter, so loop the wait block */
680 WaitBlock
->Last
= WaitBlock
;
682 /* Point to our wait block */
683 NewValue
.Value
= (OldValue
.Value
& (EX_PUSH_LOCK_MULTIPLE_SHARED
|
684 EX_PUSH_LOCK_WAKING
)) |
685 EX_PUSH_LOCK_WAITING
|
686 PtrToUlong(&WaitBlock
);
690 ASSERT(NewValue
.Waiting
);
693 /* Setup the Debug Wait Block */
694 WaitBlock
->Signaled
= 0;
695 WaitBlock
->OldValue
= OldValue
;
696 WaitBlock
->NewValue
= NewValue
;
697 WaitBlock
->PushLock
= PushLock
;
700 /* Write the new value */
701 if (InterlockedCompareExchangePointer(PushLock
,
703 OldValue
.Ptr
) != OldValue
.Ptr
)
710 /* Update the value now */
713 /* Check if the pushlock needed waking */
716 /* Scan the Waiters and Wake PushLocks */
717 ExpOptimizePushLockList(PushLock
, OldValue
);
720 /* Set up the Wait Gate */
721 KeInitializeGate(&WaitBlock
->WakeGate
);
723 /* Now spin on the push lock if necessary */
724 i
= ExPushLockSpinCount
;
725 if ((i
) && (WaitBlock
->Flags
& EX_PUSH_LOCK_WAITING
))
728 while (--i
) YieldProcessor();
731 /* Now try to remove the wait bit */
732 if (InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
734 /* Fast-path did not work, we need to do a full wait */
735 KeWaitForGate(&WaitBlock
->WakeGate
, WrPushLock
, KernelMode
);
736 ASSERT(WaitBlock
->Signaled
);
739 /* We shouldn't be shared anymore */
740 ASSERT((WaitBlock
->ShareCount
== 0));
746 * @name ExfReleasePushLock
749 * The ExReleasePushLockExclusive routine releases a previously
750 * exclusively acquired PushLock.
753 * Pointer to a previously acquired pushlock.
757 * @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
758 * This macro should usually be paired up with KeLeaveCriticalRegion.
763 ExfReleasePushLock(PEX_PUSH_LOCK PushLock
)
765 EX_PUSH_LOCK OldValue
= *PushLock
;
766 EX_PUSH_LOCK NewValue
;
767 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
;
770 ASSERT(OldValue
.Locked
);
772 /* Check if someone is waiting on the lock */
773 if (!OldValue
.Waiting
)
775 /* Nobody is waiting on it, so we'll try a quick release */
778 /* Check if it's shared */
779 if (OldValue
.Shared
> 1)
781 /* Write the Old Value but decrease share count */
787 /* Simply clear the lock */
791 /* Write the New Value */
792 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
795 if (NewValue
.Value
== OldValue
.Value
)
797 /* No waiters left, we're done */
801 /* Did it enter a wait state? */
803 if (NewValue
.Waiting
) break;
807 /* Ok, we do know someone is waiting on it. Are there more then one? */
808 if (OldValue
.MultipleShared
)
810 /* Find the last Wait Block */
811 for (WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)OldValue
.Ptr
&
812 ~EX_PUSH_LOCK_PTR_BITS
);
814 WaitBlock
= WaitBlock
->Next
);
816 /* Make sure the Share Count is above 0 */
817 if (WaitBlock
->ShareCount
)
819 /* This shouldn't be an exclusive wait block */
820 ASSERT(WaitBlock
->Flags
&EX_PUSH_LOCK_FLAGS_EXCLUSIVE
);
822 /* Do the decrease and check if the lock isn't shared anymore */
823 if (InterlockedExchangeAdd(&WaitBlock
->ShareCount
, -1))
825 /* Someone is still holding the lock */
832 * If nobody was waiting on the block, then we possibly reduced the number
833 * of times the pushlock was shared, and we unlocked it.
834 * If someone was waiting, and more then one person is waiting, then we
835 * reduced the number of times the pushlock is shared in the wait block.
836 * Therefore, at this point, we can now 'satisfy' the wait.
840 /* Now we need to see if it's waking */
843 /* Remove the lock and multiple shared bits */
844 NewValue
.Value
= OldValue
.Value
;
845 NewValue
.MultipleShared
= FALSE
;
846 NewValue
.Locked
= FALSE
;
849 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
851 /* Write the new value */
852 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
855 if (NewValue
.Value
== OldValue
.Value
) break;
857 /* The value changed, try the unlock again */
862 /* Remove the lock and multiple shared bits */
863 NewValue
.Value
= OldValue
.Value
;
864 NewValue
.MultipleShared
= FALSE
;
865 NewValue
.Locked
= FALSE
;
867 /* It's not already waking, so add the wake bit */
868 NewValue
.Waking
= TRUE
;
871 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
873 /* Write the new value */
874 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
877 if (NewValue
.Value
!= OldValue
.Value
) continue;
879 /* The write was successful. The pushlock is Unlocked and Waking */
880 ExfWakePushLock(PushLock
, NewValue
);
890 * @name ExfReleasePushLockShared
893 * The ExfReleasePushLockShared macro releases a previously acquired PushLock.
896 * Pointer to a previously acquired pushlock.
900 * @remarks Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
901 * This macro should usually be paired up with KeLeaveCriticalRegion.
906 ExfReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
908 EX_PUSH_LOCK OldValue
= *PushLock
;
909 EX_PUSH_LOCK NewValue
;
910 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
;
912 /* Check if someone is waiting on the lock */
913 if (!OldValue
.Waiting
)
915 /* Nobody is waiting on it, so we'll try a quick release */
918 /* Check if it's shared */
919 if (OldValue
.Shared
> 1)
921 /* Write the Old Value but decrease share count */
927 /* Simply clear the lock */
931 /* Write the New Value */
932 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
935 if (NewValue
.Value
== OldValue
.Value
)
937 /* No waiters left, we're done */
941 /* Did it enter a wait state? */
943 if (NewValue
.Waiting
) break;
947 /* Ok, we do know someone is waiting on it. Are there more then one? */
948 if (OldValue
.MultipleShared
)
950 /* Find the last Wait Block */
951 for (WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)((ULONG_PTR
)OldValue
.Ptr
&
952 ~EX_PUSH_LOCK_PTR_BITS
);
954 WaitBlock
= WaitBlock
->Next
);
957 ASSERT(WaitBlock
->ShareCount
> 0);
958 ASSERT(WaitBlock
->Flags
&EX_PUSH_LOCK_FLAGS_EXCLUSIVE
);
960 /* Do the decrease and check if the lock isn't shared anymore */
961 if (InterlockedExchangeAdd(&WaitBlock
->ShareCount
, -1)) goto quit
;
965 * If nobody was waiting on the block, then we possibly reduced the number
966 * of times the pushlock was shared, and we unlocked it.
967 * If someone was waiting, and more then one person is waiting, then we
968 * reduced the number of times the pushlock is shared in the wait block.
969 * Therefore, at this point, we can now 'satisfy' the wait.
973 /* Now we need to see if it's waking */
976 /* Remove the lock and multiple shared bits */
977 NewValue
.Value
= OldValue
.Value
;
978 NewValue
.MultipleShared
= FALSE
;
979 NewValue
.Locked
= FALSE
;
982 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
984 /* Write the new value */
985 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
988 if (NewValue
.Value
== OldValue
.Value
) break;
990 /* The value changed, try the unlock again */
995 /* Remove the lock and multiple shared bits */
996 NewValue
.Value
= OldValue
.Value
;
997 NewValue
.MultipleShared
= FALSE
;
998 NewValue
.Locked
= FALSE
;
1000 /* It's not already waking, so add the wake bit */
1001 NewValue
.Waking
= TRUE
;
1004 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
1006 /* Write the new value */
1007 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
1010 if (NewValue
.Value
!= OldValue
.Value
) continue;
1012 /* The write was successful. The pushlock is Unlocked and Waking */
1013 ExfWakePushLock(PushLock
, NewValue
);
1023 * ExfReleasePushLockExclusive
1024 * @implemented NT5.2
1026 * The ExfReleasePushLockExclusive routine releases a previously
1027 * exclusively acquired PushLock.
1030 * Pointer to a previously acquired pushlock.
1034 * @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1035 * This macro should usually be paired up with KeLeaveCriticalRegion.
1040 ExfReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1042 EX_PUSH_LOCK NewValue
, WakeValue
;
1043 EX_PUSH_LOCK OldValue
= *PushLock
;
1045 /* Loop until we can change */
1049 ASSERT(OldValue
.Locked
);
1050 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1052 /* Check if it's waiting and not yet waking */
1053 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1055 /* Remove the lock bit, and add the wake bit */
1056 NewValue
.Value
= (OldValue
.Value
&~ EX_PUSH_LOCK_LOCK
) |
1057 EX_PUSH_LOCK_WAKING
;
1060 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
);
1062 /* Write the New Value. Save our original value for waking */
1063 WakeValue
= NewValue
;
1064 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
1068 /* Check if the value changed behind our back */
1069 if (NewValue
.Value
!= OldValue
.Value
)
1071 /* Wake the Pushlock */
1072 ExfWakePushLock(PushLock
, WakeValue
);
1078 /* A simple unlock */
1079 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_LOCK
;
1082 ASSERT(NewValue
.Waking
&& !NewValue
.Waiting
);
1084 /* Write the New Value */
1085 NewValue
.Ptr
= InterlockedCompareExchangePointer(PushLock
,
1089 /* Check if the value changed behind our back */
1090 if (NewValue
.Value
== OldValue
.Value
) break;
1094 OldValue
= NewValue
;
1099 * @name ExfTryToWakePushLock
1100 * @implemented NT5.2
1102 * The ExfTryToWakePushLock attemps to wake a waiting pushlock.
1105 * Pointer to a PushLock which is in the wait state.
1109 * @remarks The pushlock must be in a wait state and must not be already waking.
1114 ExfTryToWakePushLock(PEX_PUSH_LOCK PushLock
)
1116 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
;
1119 * If the Pushlock is not waiting on anything, or if it's already waking up
1120 * and locked, don't do anything
1122 if (!(OldValue
.Value
== (EX_PUSH_LOCK_WAKING
| EX_PUSH_LOCK_LOCK
)) &&
1125 /* Make it Waking */
1126 NewValue
= OldValue
;
1127 NewValue
.Waking
= TRUE
;
1129 /* Write the New Value */
1130 if (InterlockedCompareExchangePointer(PushLock
,
1132 OldValue
.Ptr
) == OldValue
.Ptr
)
1134 /* Wake the Pushlock */
1135 ExfWakePushLock(PushLock
, NewValue
);
1141 * @name ExfUnblockPushLock
1142 * @implemented NT5.1
1144 * The ExfUnblockPushLock routine unblocks a previously blocked PushLock.
1147 * Pointer to a previously blocked PushLock.
1151 * @remarks Callers of ExfUnblockPushLock can be running at any IRQL.
1156 ExfUnblockPushLock(PEX_PUSH_LOCK PushLock
,
1157 PVOID CurrentWaitBlock
)
1159 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, NextWaitBlock
;
1160 KIRQL OldIrql
= DISPATCH_LEVEL
;
1162 /* Get the wait block and erase the previous one */
1163 WaitBlock
= InterlockedExchangePointer(&PushLock
->Ptr
, NULL
);
1166 /* Check if there is a linked pushlock and raise IRQL appropriately */
1167 if (WaitBlock
->Next
) KeRaiseIrql(DISPATCH_LEVEL
, &OldIrql
);
1169 /* Start block loop */
1172 /* Get the next block */
1173 NextWaitBlock
= WaitBlock
->Next
;
1175 /* Remove the wait flag from the Wait block */
1176 if (InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
1178 /* Nobody removed the flag before us, so signal the event */
1179 KeSetEventBoostPriority(&WaitBlock
->WakeEvent
, NULL
);
1182 /* Check if there was a next block */
1183 if (!NextWaitBlock
) break;
1186 /* Lower IRQL if needed */
1187 if (OldIrql
!= DISPATCH_LEVEL
) KeLowerIrql(OldIrql
);
1190 /* Check if we got a wait block that's pending */
1191 if ((CurrentWaitBlock
) &&
1192 (((PEX_PUSH_LOCK_WAIT_BLOCK
)CurrentWaitBlock
)->Flags
&
1193 EX_PUSH_LOCK_FLAGS_WAIT
))
1195 /* Wait for the pushlock to be unblocked */
1196 ExWaitForUnblockPushLock(PushLock
, CurrentWaitBlock
);