2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/rundown.c
5 * PURPOSE: Rundown and Cache-Aware Rundown Protection
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
10 /* INCLUDES *****************************************************************/
16 /* FUNCTIONS *****************************************************************/
19 * @name ExfAcquireRundownProtection
22 * The ExfAcquireRundownProtection routine acquires rundown protection for
23 * the specified descriptor.
26 * Pointer to a rundown reference descriptor.
28 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
30 * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
35 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
37 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
39 /* Loop until successfully incremented the counter */
42 /* Make sure a rundown is not active */
43 if (Value
& EX_RUNDOWN_ACTIVE
) return FALSE
;
46 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
48 /* Change the value */
49 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
50 if (NewValue
== Value
) return TRUE
;
58 * @name ExfAcquireRundownProtectionEx
61 * The ExfAcquireRundownProtectionEx routine acquires multiple rundown
62 * protection references for the specified descriptor.
65 * Pointer to a rundown reference descriptor.
68 * Number of times to reference the descriptor.
70 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
72 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
77 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef
,
80 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
82 /* Loop until successfully incremented the counter */
85 /* Make sure a rundown is not active */
86 if (Value
& EX_RUNDOWN_ACTIVE
) return FALSE
;
89 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
* Count
;
91 /* Change the value */
92 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
93 if (NewValue
== Value
) return TRUE
;
95 /* Update the value */
101 * @name ExfInitializeRundownProtection
104 * The ExfInitializeRundownProtection routine initializes a rundown
105 * protection descriptor.
108 * Pointer to a rundown reference descriptor.
112 * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
117 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
119 /* Set the count to zero */
124 * @name ExfReInitializeRundownProtection
127 * The ExfReInitializeRundownProtection routine re-initializes a rundown
128 * protection descriptor.
131 * Pointer to a rundown reference descriptor.
135 * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
140 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
145 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
147 /* Reset the count */
148 ExpSetRundown(RunRef
, 0);
152 * @name ExfRundownCompleted
155 * The ExfRundownCompleted routine completes the rundown of the specified
156 * descriptor by setting the active bit.
159 * Pointer to a rundown reference descriptor.
163 * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
168 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
173 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
175 /* Mark the counter as active */
176 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
180 * @name ExfReleaseRundownProtection
183 * The ExfReleaseRundownProtection routine releases the rundown protection
184 * reference for the specified descriptor.
187 * Pointer to a rundown reference descriptor.
191 * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
196 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
198 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
199 PEX_RUNDOWN_WAIT_BLOCK WaitBlock
;
201 /* Loop until successfully incremented the counter */
204 /* Check if rundown is not active */
205 if (!(Value
& EX_RUNDOWN_ACTIVE
))
208 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
210 /* Get the new value */
211 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
213 /* Change the value */
214 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
215 if (NewValue
== Value
) break;
222 /* Get the wait block */
223 WaitBlock
= (PEX_RUNDOWN_WAIT_BLOCK
)(Value
& ~EX_RUNDOWN_ACTIVE
);
224 ASSERT((WaitBlock
->Count
> 0) || (KeNumberProcessors
> 1));
226 /* Remove the one count */
227 if (!InterlockedDecrementSizeT(&WaitBlock
->Count
))
229 /* We're down to 0 now, so signal the event */
230 KeSetEvent(&WaitBlock
->WakeEvent
, IO_NO_INCREMENT
, FALSE
);
240 * @name ExfReleaseRundownProtectionEx
243 * The ExfReleaseRundownProtectionEx routine releases multiple rundown
244 * protection references for the specified descriptor.
247 * Pointer to a rundown reference descriptor.
250 * Number of times to dereference the descriptor.
254 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
259 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef
,
262 ULONG_PTR Value
= RunRef
->Count
, NewValue
;
263 PEX_RUNDOWN_WAIT_BLOCK WaitBlock
;
265 /* Loop until successfully incremented the counter */
268 /* Check if rundown is not active */
269 if (!(Value
& EX_RUNDOWN_ACTIVE
))
272 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
* Count
) ||
273 (KeNumberProcessors
> 1));
275 /* Get the new value */
276 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
* Count
;
278 /* Change the value */
279 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
280 if (NewValue
== Value
) break;
287 /* Get the wait block */
288 WaitBlock
= (PEX_RUNDOWN_WAIT_BLOCK
)(Value
& ~EX_RUNDOWN_ACTIVE
);
289 ASSERT((WaitBlock
->Count
>= Count
) || (KeNumberProcessors
> 1));
291 /* Remove the counts */
292 if (InterlockedExchangeAddSizeT(&WaitBlock
->Count
,
293 -(LONG
)Count
) == (LONG
)Count
)
295 /* We're down to 0 now, so signal the event */
296 KeSetEvent(&WaitBlock
->WakeEvent
, IO_NO_INCREMENT
, FALSE
);
306 * @name ExfWaitForRundownProtectionRelease
309 * The ExfWaitForRundownProtectionRelease routine waits until the specified
310 * rundown descriptor has been released.
313 * Pointer to a rundown reference descriptor.
317 * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
318 * at IRQL <= APC_LEVEL.
323 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
325 ULONG_PTR Value
, Count
, NewValue
;
326 EX_RUNDOWN_WAIT_BLOCK WaitBlock
;
327 PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer
;
331 /* Set the active bit */
332 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
333 if ((Value
== 0) || (Value
== EX_RUNDOWN_ACTIVE
)) return;
335 /* No event for now */
337 WaitBlockPointer
= (PEX_RUNDOWN_WAIT_BLOCK
)((ULONG_PTR
)&WaitBlock
|
340 /* Start waitblock set loop */
344 Count
= Value
>> EX_RUNDOWN_COUNT_SHIFT
;
346 /* If the count is over one and we don't have en event yet, create it */
347 if ((Count
) && !(Event
))
349 /* Initialize the event */
350 KeInitializeEvent(&WaitBlock
.WakeEvent
,
351 SynchronizationEvent
,
354 /* Set the pointer */
355 Event
= &WaitBlock
.WakeEvent
;
359 WaitBlock
.Count
= Count
;
361 /* Now set the pointer */
362 NewValue
= ExpChangeRundown(RunRef
, (ULONG_PTR
)WaitBlockPointer
, Value
);
363 if (NewValue
== Value
) break;
367 ASSERT((Value
& EX_RUNDOWN_ACTIVE
) == 0);
370 /* If the count was 0, we're done */
373 /* Wait for whoever needs to release to notify us */
374 KeWaitForSingleObject(Event
, Executive
, KernelMode
, FALSE
, NULL
);
375 ASSERT(WaitBlock
.Count
== 0);
378 /* FIXME: STUBS **************************************************************/
385 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
387 PEX_RUNDOWN_REF RunRef
;
389 RunRef
= ExGetRunRefForCurrentProcessor(RunRefCacheAware
);
390 return _ExAcquireRundownProtection(RunRef
);
394 * @unimplemented NT5.2
398 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
401 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware
);
402 DBG_UNREFERENCED_PARAMETER(Count
);
412 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
414 PEX_RUNDOWN_REF RunRef
;
416 RunRef
= ExGetRunRefForCurrentProcessor(RunRefCacheAware
);
417 return _ExReleaseRundownProtection(RunRef
);
421 * @unimplemented NT5.2
425 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
428 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware
);
429 DBG_UNREFERENCED_PARAMETER(Count
);
434 * @unimplemented NT5.2
438 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
440 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware
);
445 * @unimplemented NT5.2
449 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
451 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware
);
456 * @unimplemented NT5.2
460 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
462 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware
);
469 PEX_RUNDOWN_REF_CACHE_AWARE
471 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType
,
475 PEX_RUNDOWN_REF RunRef
;
476 ULONG RunRefSize
, Count
, Offset
;
477 PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
;
481 /* Allocate the master structure */
482 RunRefCacheAware
= ExAllocatePoolWithTag(PoolType
, sizeof(EX_RUNDOWN_REF_CACHE_AWARE
), Tag
);
483 if (RunRefCacheAware
== NULL
)
488 /* Compute the size of each runref */
489 RunRefCacheAware
->Number
= KeNumberProcessors
;
490 if (KeNumberProcessors
<= 1)
492 RunRefSize
= sizeof(EX_RUNDOWN_REF
);
496 RunRefSize
= KeGetRecommendedSharedDataAlignment();
497 ASSERT((RunRefSize
& (RunRefSize
- 1)) == 0);
500 /* It must at least hold a EX_RUNDOWN_REF structure */
501 ASSERT(sizeof(EX_RUNDOWN_REF
) <= RunRefSize
);
502 RunRefCacheAware
->RunRefSize
= RunRefSize
;
504 /* Allocate our runref pool */
505 PoolToFree
= ExAllocatePoolWithTag(PoolType
, RunRefSize
* RunRefCacheAware
->Number
, Tag
);
506 if (PoolToFree
== NULL
)
508 ExFreePoolWithTag(RunRefCacheAware
, Tag
);
512 /* On SMP, check for alignment */
513 if (RunRefCacheAware
->Number
> 1)
515 /* FIXME: properly align run refs */
519 RunRefCacheAware
->RunRefs
= PoolToFree
;
520 RunRefCacheAware
->PoolToFree
= PoolToFree
;
522 /* And initialize runref */
523 if (RunRefCacheAware
->Number
!= 0)
525 for (Count
= 0; Count
< RunRefCacheAware
->Number
; ++Count
)
527 Offset
= RunRefCacheAware
->RunRefSize
* Count
;
528 RunRef
= (PEX_RUNDOWN_REF
)((ULONG_PTR
)RunRefCacheAware
->RunRefs
+ Offset
);
533 return RunRefCacheAware
;
541 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
)
546 * This is to be called for RunRefCacheAware that were allocated with
547 * ExAllocateCacheAwareRundownProtection and not for user-allocated
550 ASSERT(RunRefCacheAware
->PoolToFree
!= (PVOID
)0xBADCA11);
552 /* We don't know the tag that as used for allocation */
553 ExFreePoolWithTag(RunRefCacheAware
->PoolToFree
, 0);
554 ExFreePoolWithTag(RunRefCacheAware
, 0);
562 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
566 PEX_RUNDOWN_REF RunRef
;
567 ULONG Count
, RunRefSize
, Offset
;
571 /* Get the user allocate pool for runrefs */
572 Pool
= (PVOID
)((ULONG_PTR
)RunRefCacheAware
+ sizeof(EX_RUNDOWN_REF_CACHE_AWARE
));
574 /* By default a runref is structure-sized */
575 RunRefSize
= sizeof(EX_RUNDOWN_REF
);
578 * If we just have enough room for a single runref, deduce were on a single
581 if (Size
== sizeof(EX_RUNDOWN_REF_CACHE_AWARE
) + sizeof(EX_RUNDOWN_REF
))
587 /* FIXME: Properly align on SMP */
591 /* Initialize the structure */
592 RunRefCacheAware
->RunRefs
= Pool
;
593 RunRefCacheAware
->RunRefSize
= RunRefSize
;
594 RunRefCacheAware
->Number
= Count
;
596 /* There is no allocated pool! */
597 RunRefCacheAware
->PoolToFree
= (PVOID
)0xBADCA11u
;
599 /* Initialize runref */
600 if (RunRefCacheAware
->Number
!= 0)
602 for (Count
= 0; Count
< RunRefCacheAware
->Number
; ++Count
)
604 Offset
= RunRefCacheAware
->RunRefSize
* Count
;
605 RunRef
= (PEX_RUNDOWN_REF
)((ULONG_PTR
)RunRefCacheAware
->RunRefs
+ Offset
);
616 ExSizeOfRundownProtectionCacheAware(VOID
)
622 /* Compute the needed size for runrefs */
623 if (KeNumberProcessors
<= 1)
625 Size
= sizeof(EX_RUNDOWN_REF
);
629 /* We +1, to have enough room for alignment */
630 Size
= (KeNumberProcessors
+ 1) * KeGetRecommendedSharedDataAlignment();
633 /* Return total size (master structure and runrefs) */
634 return Size
+ sizeof(EX_RUNDOWN_REF_CACHE_AWARE
);