2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/pushlock.c
5 * PURPOSE: Pushlock and Cache-Aware Pushlock Implementation
6 * PROGRAMMER: Alex Ionescu (alex.ionescu@reactos.com)
9 /* INCLUDES *****************************************************************/
15 /* DATA **********************************************************************/
17 ULONG ExPushLockSpinCount
= 0;
22 /* PRIVATE FUNCTIONS *********************************************************/
25 #define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)
27 #define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)
31 * @name ExpInitializePushLocks
33 * The ExpInitializePushLocks routine initialized Pushlock support.
39 * @remarks The ExpInitializePushLocks routine sets up the spin on SMP machines.
45 ExpInitializePushLocks(VOID
)
48 /* Initialize an internal 1024-iteration spin for MP CPUs */
49 if (KeNumberProcessors
> 1)
50 ExPushLockSpinCount
= 1024;
55 * @name ExfWakePushLock
57 * The ExfWakePushLock routine wakes a Pushlock that is in the waiting
61 * Pointer to a pushlock that is waiting.
64 * Last known value of the pushlock before this routine was called.
68 * @remarks This is an internal routine; do not call it manually. Only the system
69 * can properly know if the pushlock is ready to be awakened or not.
70 * External callers should use ExfTrytoWakePushLock.
75 ExfWakePushLock(PEX_PUSH_LOCK PushLock
,
76 EX_PUSH_LOCK OldValue
)
78 EX_PUSH_LOCK NewValue
;
79 PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock
, FirstWaitBlock
, LastWaitBlock
;
80 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
;
83 /* Start main wake loop */
87 ASSERT(!OldValue
.MultipleShared
);
89 /* Check if it's locked */
90 while (OldValue
.Locked
)
92 /* It's not waking anymore */
93 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_WAKING
;
96 ASSERT(!NewValue
.Waking
);
97 ASSERT(NewValue
.Locked
);
98 ASSERT(NewValue
.Waiting
);
100 /* Write the New Value */
101 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
104 if (NewValue
.Value
== OldValue
.Value
) return;
106 /* Someone changed the value behind our back, update it*/
110 /* Save the First Block */
111 FirstWaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(OldValue
.Value
&
112 ~EX_PUSH_LOCK_PTR_BITS
);
113 WaitBlock
= FirstWaitBlock
;
115 /* Try to find the last block */
118 /* Get the last wait block */
119 LastWaitBlock
= WaitBlock
->Last
;
121 /* Check if we found it */
125 WaitBlock
= LastWaitBlock
;
129 /* Save the previous block */
130 PreviousWaitBlock
= WaitBlock
;
132 /* Move to next block */
133 WaitBlock
= WaitBlock
->Next
;
135 /* Save the previous block */
136 WaitBlock
->Previous
= PreviousWaitBlock
;
139 /* Check if the last Wait Block is not Exclusive or if it's the only one */
140 PreviousWaitBlock
= WaitBlock
->Previous
;
141 if (!(WaitBlock
->Flags
& EX_PUSH_LOCK_FLAGS_EXCLUSIVE
) ||
142 !(PreviousWaitBlock
))
144 /* Destroy the pushlock */
146 ASSERT(!NewValue
.Waking
);
148 /* Write the New Value */
149 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
152 if (NewValue
.Value
== OldValue
.Value
) break;
154 /* Someone changed the value behind our back, update it*/
159 /* Link the wait blocks */
160 FirstWaitBlock
->Last
= PreviousWaitBlock
;
161 WaitBlock
->Previous
= NULL
;
164 ASSERT(FirstWaitBlock
!= WaitBlock
);
165 ASSERT(PushLock
->Waiting
);
167 /* Remove waking bit from pushlock */
168 InterlockedAndPointer(&PushLock
->Value
, ~EX_PUSH_LOCK_WAKING
);
175 /* Check if there's a previous block */
176 OldIrql
= DISPATCH_LEVEL
;
177 if (WaitBlock
->Previous
)
179 /* Raise to Dispatch */
180 KeRaiseIrql(DISPATCH_LEVEL
, &OldIrql
);
186 /* Get the previous Wait block */
187 PreviousWaitBlock
= WaitBlock
->Previous
;
190 ASSERT(!WaitBlock
->Signaled
);
193 /* We are about to get signaled */
194 WaitBlock
->Signaled
= TRUE
;
197 /* Set the Wait Bit in the Wait Block */
198 if (!InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
200 /* Nobody signaled us, so do it */
201 KeSignalGateBoostPriority(&WaitBlock
->WakeGate
);
204 /* Set the wait block and check if there still is one to loop*/
205 WaitBlock
= PreviousWaitBlock
;
206 if (!WaitBlock
) break;
209 /* Check if we have to lower back the IRQL */
210 if (OldIrql
!= DISPATCH_LEVEL
) KeLowerIrql(OldIrql
);
214 * @name ExpOptimizePushLockList
216 * The ExpOptimizePushLockList routine optimizes the list of waiters
217 * associated to a pushlock's wait block.
220 * Pointer to a pushlock whose waiter list needs to be optimized.
223 * Last known value of the pushlock before this routine was called.
227 * @remarks At the end of the optimization, the pushlock will also be wakened.
232 ExpOptimizePushLockList(PEX_PUSH_LOCK PushLock
,
233 EX_PUSH_LOCK OldValue
)
235 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, LastWaitBlock
, PreviousWaitBlock
, FirstWaitBlock
;
236 EX_PUSH_LOCK NewValue
;
238 /* Start main loop */
241 /* Check if we've been unlocked */
242 if (!OldValue
.Locked
)
244 /* Wake us up and leave */
245 ExfWakePushLock(PushLock
, OldValue
);
249 /* Get the wait block */
250 WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(OldValue
.Value
&
251 ~EX_PUSH_LOCK_PTR_BITS
);
253 /* Loop the blocks */
254 FirstWaitBlock
= WaitBlock
;
257 /* Get the last wait block */
258 LastWaitBlock
= WaitBlock
->Last
;
261 /* Set this as the new last block, we're done */
262 FirstWaitBlock
->Last
= LastWaitBlock
;
267 PreviousWaitBlock
= WaitBlock
;
269 /* Get the next block */
270 WaitBlock
= WaitBlock
->Next
;
272 /* Save the previous */
273 WaitBlock
->Previous
= PreviousWaitBlock
;
276 /* Remove the wake bit */
277 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_WAKING
;
280 ASSERT(NewValue
.Locked
);
281 ASSERT(!NewValue
.Waking
);
283 /* Update the value */
284 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
288 /* If we updated correctly, leave */
289 if (NewValue
.Value
== OldValue
.Value
) break;
297 * @name ExTimedWaitForUnblockPushLock
299 * The ExTimedWaitForUnblockPushLock routine waits for a pushlock
300 * to be unblocked, for a specified internal.
303 * Pointer to a pushlock whose waiter list needs to be optimized.
306 * Pointer to the pushlock's wait block.
309 * Amount of time to wait for this pushlock to be unblocked.
311 * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
312 * code returned by KeWaitForSingleObject.
314 * @remarks If the wait fails, then a manual unblock is attempted.
319 ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock
,
321 IN PLARGE_INTEGER Timeout
)
325 /* Initialize the wait event */
326 KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->WakeEvent
,
327 SynchronizationEvent
,
331 /* Spin on the push lock if necessary */
332 if (ExPushLockSpinCount
)
334 ULONG i
= ExPushLockSpinCount
;
338 /* Check if we got lucky and can leave early */
339 if (!(*(volatile LONG
*)&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->Flags
& EX_PUSH_LOCK_WAITING
))
340 return STATUS_SUCCESS
;
347 /* Now try to remove the wait bit */
348 if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->Flags
,
349 EX_PUSH_LOCK_FLAGS_WAIT_V
))
351 /* Nobody removed it already, let's do a full wait */
352 Status
= KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK
)WaitBlock
)->
358 /* Check if the wait was satisfied */
359 if (Status
!= STATUS_SUCCESS
)
361 /* Try unblocking the pushlock if it was not */
362 ExfUnblockPushLock(PushLock
, WaitBlock
);
367 /* Someone beat us to it, no need to wait */
368 Status
= STATUS_SUCCESS
;
376 * @name ExWaitForUnblockPushLock
378 * The ExWaitForUnblockPushLock routine waits for a pushlock
379 * to be unblocked, for a specified internal.
382 * Pointer to a pushlock whose waiter list needs to be optimized.
385 * Pointer to the pushlock's wait block.
387 * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
388 * code returned by KeWaitForSingleObject.
390 * @remarks If the wait fails, then a manual unblock is attempted.
395 ExWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock
,
398 /* Call the timed function with no timeout */
399 ExTimedWaitForUnblockPushLock(PushLock
, WaitBlock
, NULL
);
403 * @name ExBlockPushLock
405 * The ExBlockPushLock routine blocks a pushlock.
408 * Pointer to a pushlock whose waiter list needs to be optimized.
411 * Pointer to the pushlock's wait block.
420 ExBlockPushLock(PEX_PUSH_LOCK PushLock
,
423 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
= pWaitBlock
;
424 EX_PUSH_LOCK NewValue
, OldValue
;
426 /* Detect invalid wait block alignment */
427 ASSERT(((ULONG_PTR
)pWaitBlock
& 0xF) == 0);
429 /* Set the waiting bit */
430 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_WAIT
;
432 /* Get the old value */
433 OldValue
= *PushLock
;
435 /* Start block loop */
438 /* Link the wait blocks */
439 WaitBlock
->Next
= OldValue
.Ptr
;
441 /* Set the new wait block value */
442 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
445 if (OldValue
.Ptr
== NewValue
.Ptr
) break;
447 /* Try again with the new value */
452 /* PUBLIC FUNCTIONS **********************************************************/
455 * @name ExAcquirePushLockExclusive
458 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
461 * Pointer to the pushlock which is to be acquired.
465 * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
466 * This macro should usually be paired up with KeAcquireCriticalRegion.
471 ExfAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
473 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
, TempValue
;
475 DEFINE_WAIT_BLOCK(WaitBlock
);
477 /* Start main loop */
480 /* Check if it's unlocked */
481 if (!OldValue
.Locked
)
484 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
485 ASSERT(NewValue
.Locked
);
487 /* Set the new value */
488 if (InterlockedCompareExchangePointer(&PushLock
->Ptr
,
490 OldValue
.Ptr
) != OldValue
.Ptr
)
493 OldValue
= *PushLock
;
497 /* Break out of the loop */
502 /* We'll have to create a Waitblock */
503 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_EXCLUSIVE
|
504 EX_PUSH_LOCK_FLAGS_WAIT
;
505 WaitBlock
->Previous
= NULL
;
508 /* Check if there is already a waiter */
509 if (OldValue
.Waiting
)
511 /* Nobody is the last waiter yet */
512 WaitBlock
->Last
= NULL
;
514 /* We are an exclusive waiter */
515 WaitBlock
->ShareCount
= 0;
517 /* Set the current Wait Block pointer */
518 WaitBlock
->Next
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(
519 OldValue
.Value
&~ EX_PUSH_LOCK_PTR_BITS
);
522 NewValue
.Value
= (OldValue
.Value
& EX_PUSH_LOCK_MULTIPLE_SHARED
) |
524 EX_PUSH_LOCK_WAKING
|
525 EX_PUSH_LOCK_WAITING
|
526 (ULONG_PTR
)WaitBlock
;
528 /* Check if the pushlock was already waking */
529 if (!OldValue
.Waking
) NeedWake
= TRUE
;
533 /* We are the first waiter, so loop the wait block */
534 WaitBlock
->Last
= WaitBlock
;
536 /* Set the share count */
537 WaitBlock
->ShareCount
= (LONG
)OldValue
.Shared
;
539 /* Check if someone is sharing this pushlock */
540 if (OldValue
.Shared
> 1)
542 /* Point to our wait block */
543 NewValue
.Value
= EX_PUSH_LOCK_MULTIPLE_SHARED
|
545 EX_PUSH_LOCK_WAITING
|
546 (ULONG_PTR
)WaitBlock
;
550 /* No shared count */
551 WaitBlock
->ShareCount
= 0;
553 /* Point to our wait block */
554 NewValue
.Value
= EX_PUSH_LOCK_LOCK
|
555 EX_PUSH_LOCK_WAITING
|
556 (ULONG_PTR
)WaitBlock
;
561 /* Setup the Debug Wait Block */
562 WaitBlock
->Signaled
= 0;
563 WaitBlock
->OldValue
= OldValue
;
564 WaitBlock
->NewValue
= NewValue
;
565 WaitBlock
->PushLock
= PushLock
;
569 ASSERT(NewValue
.Waiting
);
570 ASSERT(NewValue
.Locked
);
572 /* Write the new value */
573 TempValue
= NewValue
;
574 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
577 if (NewValue
.Value
!= OldValue
.Value
)
580 OldValue
= *PushLock
;
584 /* Check if the pushlock needed waking */
587 /* Scan the Waiters and Wake PushLocks */
588 ExpOptimizePushLockList(PushLock
, TempValue
);
591 /* Set up the Wait Gate */
592 KeInitializeGate(&WaitBlock
->WakeGate
);
595 /* Now spin on the push lock if necessary */
596 if (ExPushLockSpinCount
)
598 ULONG i
= ExPushLockSpinCount
;
602 if (!(*(volatile LONG
*)&WaitBlock
->Flags
& EX_PUSH_LOCK_WAITING
))
610 /* Now try to remove the wait bit */
611 if (InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
613 /* Nobody removed it already, let's do a full wait */
614 KeWaitForGate(&WaitBlock
->WakeGate
, WrPushLock
, KernelMode
);
615 ASSERT(WaitBlock
->Signaled
);
618 /* We shouldn't be shared anymore */
619 ASSERT((WaitBlock
->ShareCount
== 0));
628 * @name ExAcquirePushLockShared
631 * The ExAcquirePushLockShared routine acquires a shared PushLock.
634 * Pointer to the pushlock which is to be acquired.
638 * @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
639 * This macro should usually be paired up with KeAcquireCriticalRegion.
644 ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
646 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
;
648 DEFINE_WAIT_BLOCK(WaitBlock
);
650 /* Start main loop */
653 /* Check if it's unlocked or if it's waiting without any sharers */
654 if (!(OldValue
.Locked
) || (!(OldValue
.Waiting
) && (OldValue
.Shared
> 0)))
656 /* Check if anyone is waiting on it */
657 if (!OldValue
.Waiting
)
659 /* Increase the share count and lock it */
660 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
665 /* Simply set the lock bit */
666 NewValue
.Value
= OldValue
.Value
| EX_PUSH_LOCK_LOCK
;
670 ASSERT(NewValue
.Locked
);
672 /* Set the new value */
673 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
676 if (NewValue
.Value
!= OldValue
.Value
)
679 OldValue
= *PushLock
;
683 /* Break out of the loop */
688 /* We'll have to create a Waitblock */
689 WaitBlock
->Flags
= EX_PUSH_LOCK_FLAGS_WAIT
;
690 WaitBlock
->ShareCount
= 0;
692 WaitBlock
->Previous
= NULL
;
694 /* Check if there is already a waiter */
695 if (OldValue
.Waiting
)
697 /* Set the current Wait Block pointer */
698 WaitBlock
->Next
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(
699 OldValue
.Value
&~ EX_PUSH_LOCK_PTR_BITS
);
701 /* Nobody is the last waiter yet */
702 WaitBlock
->Last
= NULL
;
705 NewValue
.Value
= (OldValue
.Value
& (EX_PUSH_LOCK_MULTIPLE_SHARED
|
706 EX_PUSH_LOCK_LOCK
)) |
707 EX_PUSH_LOCK_WAKING
|
708 EX_PUSH_LOCK_WAITING
|
709 (ULONG_PTR
)WaitBlock
;
711 /* Check if the pushlock was already waking */
712 if (!OldValue
.Waking
) NeedWake
= TRUE
;
716 /* We are the first waiter, so loop the wait block */
717 WaitBlock
->Last
= WaitBlock
;
719 /* Point to our wait block */
720 NewValue
.Value
= (OldValue
.Value
& EX_PUSH_LOCK_PTR_BITS
) |
721 EX_PUSH_LOCK_WAITING
|
722 (ULONG_PTR
)WaitBlock
;
726 ASSERT(NewValue
.Waiting
);
729 /* Setup the Debug Wait Block */
730 WaitBlock
->Signaled
= 0;
731 WaitBlock
->OldValue
= OldValue
;
732 WaitBlock
->NewValue
= NewValue
;
733 WaitBlock
->PushLock
= PushLock
;
736 /* Write the new value */
737 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
740 if (NewValue
.Ptr
!= OldValue
.Ptr
)
743 OldValue
= *PushLock
;
747 /* Update the value now */
750 /* Check if the pushlock needed waking */
753 /* Scan the Waiters and Wake PushLocks */
754 ExpOptimizePushLockList(PushLock
, OldValue
);
757 /* Set up the Wait Gate */
758 KeInitializeGate(&WaitBlock
->WakeGate
);
761 /* Now spin on the push lock if necessary */
762 if (ExPushLockSpinCount
)
764 ULONG i
= ExPushLockSpinCount
;
768 if (!(*(volatile LONG
*)&WaitBlock
->Flags
& EX_PUSH_LOCK_WAITING
))
776 /* Now try to remove the wait bit */
777 if (InterlockedBitTestAndReset(&WaitBlock
->Flags
, 1))
779 /* Fast-path did not work, we need to do a full wait */
780 KeWaitForGate(&WaitBlock
->WakeGate
, WrPushLock
, KernelMode
);
781 ASSERT(WaitBlock
->Signaled
);
784 /* We shouldn't be shared anymore */
785 ASSERT((WaitBlock
->ShareCount
== 0));
791 * @name ExfReleasePushLock
794 * The ExReleasePushLock routine releases a previously acquired PushLock.
798 * Pointer to a previously acquired pushlock.
802 * @remarks Callers of ExfReleasePushLock must be running at IRQL <= APC_LEVEL.
803 * This macro should usually be paired up with KeLeaveCriticalRegion.
808 ExfReleasePushLock(PEX_PUSH_LOCK PushLock
)
810 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
, WakeValue
;
811 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, LastWaitBlock
;
814 ASSERT(OldValue
.Locked
);
816 /* Start main loop */
819 /* Check if someone is waiting on the lock */
820 if (!OldValue
.Waiting
)
822 /* Check if it's shared */
823 if (OldValue
.Shared
> 1)
825 /* Write the Old Value but decrease share count */
831 /* Simply clear the lock */
835 /* Write the New Value */
836 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
839 if (NewValue
.Value
== OldValue
.Value
) return;
841 /* Did it enter a wait state? */
846 /* Ok, we do know someone is waiting on it. Are there more then one? */
847 if (OldValue
.MultipleShared
)
849 /* Get the wait block */
850 WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(OldValue
.Value
&
851 ~EX_PUSH_LOCK_PTR_BITS
);
853 /* Loop until we find the last wait block */
856 /* Get the last wait block */
857 LastWaitBlock
= WaitBlock
->Last
;
863 WaitBlock
= LastWaitBlock
;
868 WaitBlock
= WaitBlock
->Next
;
871 /* Make sure the Share Count is above 0 */
872 if (WaitBlock
->ShareCount
> 0)
874 /* This shouldn't be an exclusive wait block */
875 ASSERT(WaitBlock
->Flags
& EX_PUSH_LOCK_FLAGS_EXCLUSIVE
);
877 /* Do the decrease and check if the lock isn't shared anymore */
878 if (InterlockedDecrement(&WaitBlock
->ShareCount
) > 0) return;
883 * If nobody was waiting on the block, then we possibly reduced the number
884 * of times the pushlock was shared, and we unlocked it.
885 * If someone was waiting, and more then one person is waiting, then we
886 * reduced the number of times the pushlock is shared in the wait block.
887 * Therefore, at this point, we can now 'satisfy' the wait.
891 /* Now we need to see if it's waking */
894 /* Remove the lock and multiple shared bits */
895 NewValue
.Value
= OldValue
.Value
;
896 NewValue
.MultipleShared
= FALSE
;
897 NewValue
.Locked
= FALSE
;
900 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
902 /* Write the new value */
903 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
906 if (NewValue
.Value
== OldValue
.Value
) return;
910 /* Remove the lock and multiple shared bits */
911 NewValue
.Value
= OldValue
.Value
;
912 NewValue
.MultipleShared
= FALSE
;
913 NewValue
.Locked
= FALSE
;
915 /* It's not already waking, so add the wake bit */
916 NewValue
.Waking
= TRUE
;
919 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
921 /* Write the new value */
922 WakeValue
= NewValue
;
923 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
926 if (NewValue
.Value
!= OldValue
.Value
) continue;
928 /* The write was successful. The pushlock is Unlocked and Waking */
929 ExfWakePushLock(PushLock
, WakeValue
);
938 * @name ExfReleasePushLockShared
941 * The ExfReleasePushLockShared macro releases a previously acquired PushLock.
944 * Pointer to a previously acquired pushlock.
948 * @remarks Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
949 * This macro should usually be paired up with KeLeaveCriticalRegion.
954 ExfReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
956 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
, WakeValue
;
957 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, LastWaitBlock
;
959 /* Check if someone is waiting on the lock */
960 while (!OldValue
.Waiting
)
962 /* Check if it's shared */
963 if (OldValue
.Shared
> 1)
965 /* Write the Old Value but decrease share count */
971 /* Simply clear the lock */
975 /* Write the New Value */
976 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
979 if (NewValue
.Value
== OldValue
.Value
) return;
981 /* Did it enter a wait state? */
985 /* Ok, we do know someone is waiting on it. Are there more then one? */
986 if (OldValue
.MultipleShared
)
988 /* Get the wait block */
989 WaitBlock
= (PEX_PUSH_LOCK_WAIT_BLOCK
)(OldValue
.Value
&
990 ~EX_PUSH_LOCK_PTR_BITS
);
992 /* Loop until we find the last wait block */
995 /* Get the last wait block */
996 LastWaitBlock
= WaitBlock
->Last
;
1002 WaitBlock
= LastWaitBlock
;
1006 /* Keep searching */
1007 WaitBlock
= WaitBlock
->Next
;
1011 ASSERT(WaitBlock
->ShareCount
> 0);
1012 ASSERT(WaitBlock
->Flags
& EX_PUSH_LOCK_FLAGS_EXCLUSIVE
);
1014 /* Do the decrease and check if the lock isn't shared anymore */
1015 if (InterlockedDecrement(&WaitBlock
->ShareCount
) > 0) return;
1019 * If nobody was waiting on the block, then we possibly reduced the number
1020 * of times the pushlock was shared, and we unlocked it.
1021 * If someone was waiting, and more then one person is waiting, then we
1022 * reduced the number of times the pushlock is shared in the wait block.
1023 * Therefore, at this point, we can now 'satisfy' the wait.
1027 /* Now we need to see if it's waking */
1028 if (OldValue
.Waking
)
1030 /* Remove the lock and multiple shared bits */
1031 NewValue
.Value
= OldValue
.Value
;
1032 NewValue
.MultipleShared
= FALSE
;
1033 NewValue
.Locked
= FALSE
;
1036 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
1038 /* Write the new value */
1039 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
1042 if (NewValue
.Value
== OldValue
.Value
) return;
1046 /* Remove the lock and multiple shared bits */
1047 NewValue
.Value
= OldValue
.Value
;
1048 NewValue
.MultipleShared
= FALSE
;
1049 NewValue
.Locked
= FALSE
;
1051 /* It's not already waking, so add the wake bit */
1052 NewValue
.Waking
= TRUE
;
1055 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
&& !NewValue
.MultipleShared
);
1057 /* Write the new value */
1058 WakeValue
= NewValue
;
1059 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
1062 if (NewValue
.Value
!= OldValue
.Value
) continue;
1064 /* The write was successful. The pushlock is Unlocked and Waking */
1065 ExfWakePushLock(PushLock
, WakeValue
);
1072 * ExfReleasePushLockExclusive
1073 * @implemented NT5.2
1075 * The ExfReleasePushLockExclusive routine releases a previously
1076 * exclusively acquired PushLock.
1079 * Pointer to a previously acquired pushlock.
1083 * @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1084 * This macro should usually be paired up with KeLeaveCriticalRegion.
1089 ExfReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1091 EX_PUSH_LOCK NewValue
, WakeValue
;
1092 EX_PUSH_LOCK OldValue
= *PushLock
;
1094 /* Loop until we can change */
1098 ASSERT(OldValue
.Locked
);
1099 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1101 /* Check if it's waiting and not yet waking */
1102 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1104 /* Remove the lock bit, and add the wake bit */
1105 NewValue
.Value
= (OldValue
.Value
&~ EX_PUSH_LOCK_LOCK
) |
1106 EX_PUSH_LOCK_WAKING
;
1109 ASSERT(NewValue
.Waking
&& !NewValue
.Locked
);
1111 /* Write the New Value. Save our original value for waking */
1112 WakeValue
= NewValue
;
1113 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
1117 /* Check if the value changed behind our back */
1118 if (NewValue
.Value
== OldValue
.Value
)
1120 /* Wake the Pushlock */
1121 ExfWakePushLock(PushLock
, WakeValue
);
1127 /* A simple unlock */
1128 NewValue
.Value
= OldValue
.Value
&~ EX_PUSH_LOCK_LOCK
;
1131 ASSERT(NewValue
.Waking
&& !NewValue
.Waiting
);
1133 /* Write the New Value */
1134 NewValue
.Ptr
= InterlockedCompareExchangePointer(&PushLock
->Ptr
,
1138 /* Check if the value changed behind our back */
1139 if (NewValue
.Value
== OldValue
.Value
) break;
1143 OldValue
= NewValue
;
1148 * @name ExfTryToWakePushLock
1149 * @implemented NT5.2
1151 * The ExfTryToWakePushLock attemps to wake a waiting pushlock.
1154 * Pointer to a PushLock which is in the wait state.
1158 * @remarks The pushlock must be in a wait state and must not be already waking.
1163 ExfTryToWakePushLock(PEX_PUSH_LOCK PushLock
)
1165 EX_PUSH_LOCK OldValue
= *PushLock
, NewValue
;
1168 * If the Pushlock is not waiting on anything, or if it's already waking up
1169 * and locked, don't do anything
1171 if ((OldValue
.Waking
) || (OldValue
.Locked
) || !(OldValue
.Waiting
)) return;
1173 /* Make it Waking */
1174 NewValue
= OldValue
;
1175 NewValue
.Waking
= TRUE
;
1177 /* Write the New Value */
1178 if (InterlockedCompareExchangePointer(&PushLock
->Ptr
,
1180 OldValue
.Ptr
) == OldValue
.Ptr
)
1182 /* Wake the Pushlock */
1183 ExfWakePushLock(PushLock
, NewValue
);
1188 * @name ExfUnblockPushLock
1189 * @implemented NT5.1
1191 * The ExfUnblockPushLock routine unblocks a previously blocked PushLock.
1194 * Pointer to a previously blocked PushLock.
1198 * @remarks Callers of ExfUnblockPushLock can be running at any IRQL.
1203 ExfUnblockPushLock(PEX_PUSH_LOCK PushLock
,
1204 PVOID CurrentWaitBlock
)
1206 PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
, NextWaitBlock
;
1207 KIRQL OldIrql
= DISPATCH_LEVEL
;
1209 /* Get the wait block and erase the previous one */
1210 WaitBlock
= InterlockedExchangePointer(&PushLock
->Ptr
, NULL
);
1213 /* Check if there is a linked pushlock and raise IRQL appropriately */
1214 if (WaitBlock
->Next
) KeRaiseIrql(DISPATCH_LEVEL
, &OldIrql
);
1216 /* Start block loop */
1219 /* Get the next block */
1220 NextWaitBlock
= WaitBlock
->Next
;
1222 /* Remove the wait flag from the Wait block */
1223 if (!InterlockedBitTestAndReset(&WaitBlock
->Flags
, EX_PUSH_LOCK_FLAGS_WAIT_V
))
1225 /* Nobody removed the flag before us, so signal the event */
1226 KeSetEventBoostPriority(&WaitBlock
->WakeEvent
, NULL
);
1229 /* Try the next one */
1230 WaitBlock
= NextWaitBlock
;
1233 /* Lower IRQL if needed */
1234 if (OldIrql
!= DISPATCH_LEVEL
) KeLowerIrql(OldIrql
);
1237 /* Check if we got a wait block that's pending */
1238 if ((CurrentWaitBlock
) &&
1239 (((PEX_PUSH_LOCK_WAIT_BLOCK
)CurrentWaitBlock
)->Flags
&
1240 EX_PUSH_LOCK_FLAGS_WAIT
))
1242 /* Wait for the pushlock to be unblocked */
1243 ExWaitForUnblockPushLock(PushLock
, CurrentWaitBlock
);