2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/rundown.c
5 * PURPOSE: Rundown and Cache-Aware Rundown Protection
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
11 /* INCLUDES *****************************************************************/
17 /* FUNCTIONS *****************************************************************/
20 * @name ExfAcquireRundownProtection
23 * The ExfAcquireRundownProtection routine acquires rundown protection for
24 * the specified descriptor.
27 * Pointer to a rundown reference descriptor.
29 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
31 * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
36 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
38 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
40 /* Loop until successfully incremented the counter */
43 /* Make sure a rundown is not active */
44 if (Value
& EX_RUNDOWN_ACTIVE
) return FALSE
;
47 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
49 /* Change the value */
50 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
51 if (NewValue
== Value
) return TRUE
;
59 * @name ExfAcquireRundownProtectionEx
62 * The ExfAcquireRundownProtectionEx routine acquires multiple rundown
63 * protection references for the specified descriptor.
66 * Pointer to a rundown reference descriptor.
69 * Number of times to reference the descriptor.
71 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
73 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
78 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef
,
81 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
83 /* Loop until successfully incremented the counter */
86 /* Make sure a rundown is not active */
87 if (Value
& EX_RUNDOWN_ACTIVE
) return FALSE
;
90 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
* Count
;
92 /* Change the value */
93 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
94 if (NewValue
== Value
) return TRUE
;
96 /* Update the value */
102 * @name ExfInitializeRundownProtection
105 * The ExfInitializeRundownProtection routine initializes a rundown
106 * protection descriptor.
109 * Pointer to a rundown reference descriptor.
113 * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
118 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
120 /* Set the count to zero */
125 * @name ExfReInitializeRundownProtection
128 * The ExfReInitializeRundownProtection routine re-initializes a rundown
129 * protection descriptor.
132 * Pointer to a rundown reference descriptor.
136 * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
141 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
146 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
148 /* Reset the count */
149 ExpSetRundown(RunRef
, 0);
153 * @name ExfRundownCompleted
156 * The ExfRundownCompleted routine completes the rundown of the specified
157 * descriptor by setting the active bit.
160 * Pointer to a rundown reference descriptor.
164 * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
169 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
174 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
176 /* Mark the counter as active */
177 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
181 * @name ExfReleaseRundownProtection
184 * The ExfReleaseRundownProtection routine releases the rundown protection
185 * reference for the specified descriptor.
188 * Pointer to a rundown reference descriptor.
192 * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
197 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
199 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
200 PEX_RUNDOWN_WAIT_BLOCK WaitBlock
;
202 /* Loop until successfully incremented the counter */
205 /* Check if rundown is not active */
206 if (!(Value
& EX_RUNDOWN_ACTIVE
))
209 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
211 /* Get the new value */
212 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
214 /* Change the value */
215 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
216 if (NewValue
== Value
) break;
223 /* Get the wait block */
224 WaitBlock
= (PEX_RUNDOWN_WAIT_BLOCK
)(Value
& ~EX_RUNDOWN_ACTIVE
);
225 ASSERT((WaitBlock
->Count
> 0) || (KeNumberProcessors
> 1));
227 /* Remove the one count */
228 if (!InterlockedDecrementSizeT(&WaitBlock
->Count
))
230 /* We're down to 0 now, so signal the event */
231 KeSetEvent(&WaitBlock
->WakeEvent
, IO_NO_INCREMENT
, FALSE
);
241 * @name ExfReleaseRundownProtectionEx
244 * The ExfReleaseRundownProtectionEx routine releases multiple rundown
245 * protection references for the specified descriptor.
248 * Pointer to a rundown reference descriptor.
251 * Number of times to dereference the descriptor.
255 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
260 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef
,
263 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
264 PEX_RUNDOWN_WAIT_BLOCK WaitBlock
;
266 /* Loop until successfully incremented the counter */
269 /* Check if rundown is not active */
270 if (!(Value
& EX_RUNDOWN_ACTIVE
))
273 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
* Count
) ||
274 (KeNumberProcessors
> 1));
276 /* Get the new value */
277 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
* Count
;
279 /* Change the value */
280 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
281 if (NewValue
== Value
) break;
288 /* Get the wait block */
289 WaitBlock
= (PEX_RUNDOWN_WAIT_BLOCK
)(Value
& ~EX_RUNDOWN_ACTIVE
);
290 ASSERT((WaitBlock
->Count
>= Count
) || (KeNumberProcessors
> 1));
292 /* Remove the counts */
293 if (InterlockedExchangeAddSizeT(&WaitBlock
->Count
,
294 -(LONG
)Count
) == (LONG
)Count
)
296 /* We're down to 0 now, so signal the event */
297 KeSetEvent(&WaitBlock
->WakeEvent
, IO_NO_INCREMENT
, FALSE
);
307 * @name ExfWaitForRundownProtectionRelease
310 * The ExfWaitForRundownProtectionRelease routine waits until the specified
311 * rundown descriptor has been released.
314 * Pointer to a rundown reference descriptor.
318 * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
319 * at IRQL <= APC_LEVEL.
324 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
326 ULONG_PTR Value
, Count
, NewValue
;
327 EX_RUNDOWN_WAIT_BLOCK WaitBlock
;
328 PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer
;
332 /* Set the active bit */
333 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
334 if ((Value
== 0) || (Value
== EX_RUNDOWN_ACTIVE
)) return;
336 /* No event for now */
338 WaitBlockPointer
= (PEX_RUNDOWN_WAIT_BLOCK
)((ULONG_PTR
)&WaitBlock
|
341 /* Start waitblock set loop */
345 Count
= Value
>> EX_RUNDOWN_COUNT_SHIFT
;
347 /* If the count is over one and we don't have en event yet, create it */
348 if ((Count
) && !(Event
))
350 /* Initialize the event */
351 KeInitializeEvent(&WaitBlock
.WakeEvent
,
352 SynchronizationEvent
,
355 /* Set the pointer */
356 Event
= &WaitBlock
.WakeEvent
;
360 WaitBlock
.Count
= Count
;
362 /* Now set the pointer */
363 NewValue
= ExpChangeRundown(RunRef
, (ULONG_PTR
)WaitBlockPointer
, Value
);
364 if (NewValue
== Value
) break;
368 ASSERT((Value
& EX_RUNDOWN_ACTIVE
) == 0);
371 /* If the count was 0, we're done */
374 /* Wait for whoever needs to release to notify us */
375 KeWaitForSingleObject(Event
, Executive
, KernelMode
, FALSE
, NULL
);
376 ASSERT(WaitBlock
.Count
== 0);
384 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
386 PEX_RUNDOWN_REF RunRef
;
388 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, KeGetCurrentProcessorNumber());
389 return _ExAcquireRundownProtection(RunRef
);
397 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
400 PEX_RUNDOWN_REF RunRef
;
402 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, KeGetCurrentProcessorNumber());
403 return ExfAcquireRundownProtectionEx(RunRef
, Count
);
411 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
413 PEX_RUNDOWN_REF RunRef
;
415 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, KeGetCurrentProcessorNumber());
416 _ExReleaseRundownProtection(RunRef
);
424 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
427 PEX_RUNDOWN_REF RunRef
;
429 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, KeGetCurrentProcessorNumber());
430 ExfReleaseRundownProtectionEx(RunRef
, Count
);
438 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
440 PEX_RUNDOWN_REF RunRef
;
441 EX_RUNDOWN_WAIT_BLOCK WaitBlock
;
442 PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer
;
443 ULONG ProcCount
, Current
, Value
, OldValue
, TotalCount
;
445 ProcCount
= RunRefCacheAware
->Number
;
446 /* No proc, nothing to do */
454 WaitBlockPointer
= (PEX_RUNDOWN_WAIT_BLOCK
)((ULONG_PTR
)&WaitBlock
|
456 /* We will check all our runrefs */
457 for (Current
= 0; Current
< ProcCount
; ++Current
)
459 /* Get the runref for the proc */
460 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, Current
);
461 /* Loop for setting the wait block */
464 Value
= RunRef
->Count
;
465 ASSERT((Value
& EX_RUNDOWN_ACTIVE
) == 0);
467 /* Remove old value and set our waitblock instead */
468 OldValue
= ExpChangeRundown(RunRef
, WaitBlockPointer
, Value
);
469 if (OldValue
== Value
)
478 /* Count the deleted values */
482 /* Sanity check: we didn't overflow */
483 ASSERT((LONG
)TotalCount
>= 0);
486 /* Init the waitblock event */
487 KeInitializeEvent(&WaitBlock
.WakeEvent
,
488 SynchronizationEvent
,
491 /* Do we have to wait? If so, go ahead! */
492 if (InterlockedExchangeAddSizeT(&WaitBlock
.Count
,
493 (LONG
)TotalCount
>> EX_RUNDOWN_COUNT_SHIFT
) ==
494 -(LONG
)(TotalCount
>> EX_RUNDOWN_COUNT_SHIFT
))
496 KeWaitForSingleObject(&WaitBlock
.WakeEvent
, Executive
, KernelMode
, FALSE
, NULL
);
506 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
508 PEX_RUNDOWN_REF RunRef
;
509 ULONG ProcCount
, Current
;
511 ProcCount
= RunRefCacheAware
->Number
;
512 /* No proc, nothing to do */
518 /* We will mark all our runrefs active */
519 for (Current
= 0; Current
< ProcCount
; ++Current
)
521 /* Get the runref for the proc */
522 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, Current
);
523 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
525 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
534 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
536 PEX_RUNDOWN_REF RunRef
;
537 ULONG ProcCount
, Current
;
539 ProcCount
= RunRefCacheAware
->Number
;
540 /* No proc, nothing to do */
546 /* We will mark all our runrefs inactive */
547 for (Current
= 0; Current
< ProcCount
; ++Current
)
549 /* Get the runref for the proc */
550 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, Current
);
551 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
553 ExpSetRundown(RunRef
, 0);
560 PEX_RUNDOWN_REF_CACHE_AWARE
562 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType
,
565 PEX_RUNDOWN_REF RunRef
;
566 PVOID PoolToFree
, RunRefs
;
567 ULONG RunRefSize
, Count
, Align
;
568 PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
;
572 /* Allocate the master structure */
573 RunRefCacheAware
= ExAllocatePoolWithTag(PoolType
, sizeof(EX_RUNDOWN_REF_CACHE_AWARE
), Tag
);
574 if (RunRefCacheAware
== NULL
)
579 /* Compute the size of each runref */
580 RunRefCacheAware
->Number
= KeNumberProcessors
;
581 if (KeNumberProcessors
<= 1)
583 RunRefSize
= sizeof(EX_RUNDOWN_REF
);
587 Align
= KeGetRecommendedSharedDataAlignment();
589 ASSERT((RunRefSize
& (RunRefSize
- 1)) == 0);
592 /* It must at least hold a EX_RUNDOWN_REF structure */
593 ASSERT(sizeof(EX_RUNDOWN_REF
) <= RunRefSize
);
594 RunRefCacheAware
->RunRefSize
= RunRefSize
;
596 /* Allocate our runref pool */
597 PoolToFree
= ExAllocatePoolWithTag(PoolType
, RunRefSize
* RunRefCacheAware
->Number
, Tag
);
598 if (PoolToFree
== NULL
)
600 ExFreePoolWithTag(RunRefCacheAware
, Tag
);
604 /* On SMP, check for alignment */
605 if (RunRefCacheAware
->Number
> 1 && (ULONG_PTR
)PoolToFree
& (Align
- 1))
607 /* Not properly aligned, do it again! */
608 ExFreePoolWithTag(PoolToFree
, Tag
);
610 /* Allocate a bigger buffer to be able to align properly */
611 PoolToFree
= ExAllocatePoolWithTag(PoolType
, RunRefSize
* (RunRefCacheAware
->Number
+ 1), Tag
);
612 if (PoolToFree
== NULL
)
614 ExFreePoolWithTag(RunRefCacheAware
, Tag
);
618 RunRefs
= (PVOID
)ALIGN_UP_BY(PoolToFree
, Align
);
622 RunRefs
= PoolToFree
;
625 RunRefCacheAware
->RunRefs
= RunRefs
;
626 RunRefCacheAware
->PoolToFree
= PoolToFree
;
628 /* And initialize runref */
629 if (RunRefCacheAware
->Number
!= 0)
631 for (Count
= 0; Count
< RunRefCacheAware
->Number
; ++Count
)
633 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, Count
);
634 _ExInitializeRundownProtection(RunRef
);
638 return RunRefCacheAware
;
646 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
651 * This is to be called for RunRefCacheAware that were allocated with
652 * ExAllocateCacheAwareRundownProtection and not for user-allocated
655 ASSERT(RunRefCacheAware
->PoolToFree
!= (PVOID
)0xBADCA11);
657 /* We don't know the tag that as used for allocation */
658 ExFreePoolWithTag(RunRefCacheAware
->PoolToFree
, 0);
659 ExFreePoolWithTag(RunRefCacheAware
, 0);
667 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
671 PEX_RUNDOWN_REF RunRef
;
672 ULONG Count
, RunRefSize
, Align
;
676 /* Get the user allocate pool for runrefs */
677 Pool
= (PVOID
)((ULONG_PTR
)RunRefCacheAware
+ sizeof(EX_RUNDOWN_REF_CACHE_AWARE
));
679 /* By default a runref is structure-sized */
680 RunRefSize
= sizeof(EX_RUNDOWN_REF
);
683 * If we just have enough room for a single runref, deduce were on a single
686 if (Size
== sizeof(EX_RUNDOWN_REF_CACHE_AWARE
) + sizeof(EX_RUNDOWN_REF
))
692 /* Get alignment constraint */
693 Align
= KeGetRecommendedSharedDataAlignment();
695 /* How many runrefs given the alignment? */
697 Count
= ((Size
- sizeof(EX_RUNDOWN_REF_CACHE_AWARE
)) / Align
) - 1;
698 Pool
= (PVOID
)ALIGN_UP_BY(Pool
, Align
);
701 /* Initialize the structure */
702 RunRefCacheAware
->RunRefs
= Pool
;
703 RunRefCacheAware
->RunRefSize
= RunRefSize
;
704 RunRefCacheAware
->Number
= Count
;
706 /* There is no allocated pool! */
707 RunRefCacheAware
->PoolToFree
= (PVOID
)0xBADCA11u
;
709 /* Initialize runref */
710 if (RunRefCacheAware
->Number
!= 0)
712 for (Count
= 0; Count
< RunRefCacheAware
->Number
; ++Count
)
714 RunRef
= ExGetRunRefForGivenProcessor(RunRefCacheAware
, Count
);
715 _ExInitializeRundownProtection(RunRef
);
725 ExSizeOfRundownProtectionCacheAware(VOID
)
731 /* Compute the needed size for runrefs */
732 if (KeNumberProcessors
<= 1)
734 Size
= sizeof(EX_RUNDOWN_REF
);
738 /* We +1, to have enough room for alignment */
739 Size
= (KeNumberProcessors
+ 1) * KeGetRecommendedSharedDataAlignment();
742 /* Return total size (master structure and runrefs) */
743 return Size
+ sizeof(EX_RUNDOWN_REF_CACHE_AWARE
);