[NTOSKRNL] Simplify implementation of ExfAcquireRundownProtectionCacheAware(), ExfRel...
[reactos.git] / ntoskrnl / ex / rundown.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/rundown.c
5 * PURPOSE: Rundown and Cache-Aware Rundown Protection
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
7 * Thomas Weidenmueller
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 /* FUNCTIONS *****************************************************************/
17
18 /*++
19 * @name ExfAcquireRundownProtection
20 * @implemented NT5.1
21 *
22 * The ExfAcquireRundownProtection routine acquires rundown protection for
23 * the specified descriptor.
24 *
25 * @param RunRef
26 * Pointer to a rundown reference descriptor.
27 *
28 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
29 *
30 * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
31 *
32 *--*/
33 BOOLEAN
34 FASTCALL
35 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
36 {
37 ULONG_PTR Value = RunRef->Count, NewValue;
38
39 /* Loop until successfully incremented the counter */
40 for (;;)
41 {
42 /* Make sure a rundown is not active */
43 if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
44
45 /* Add a reference */
46 NewValue = Value + EX_RUNDOWN_COUNT_INC;
47
48 /* Change the value */
49 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
50 if (NewValue == Value) return TRUE;
51
52 /* Update it */
53 Value = NewValue;
54 }
55 }
56
57 /*++
58 * @name ExfAcquireRundownProtectionEx
59 * @implemented NT5.2
60 *
61 * The ExfAcquireRundownProtectionEx routine acquires multiple rundown
62 * protection references for the specified descriptor.
63 *
64 * @param RunRef
65 * Pointer to a rundown reference descriptor.
66 *
67 * @param Count
68 * Number of times to reference the descriptor.
69 *
70 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
71 *
72 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
73 *
74 *--*/
75 BOOLEAN
76 FASTCALL
77 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
78 IN ULONG Count)
79 {
80 ULONG_PTR Value = RunRef->Count, NewValue;
81
82 /* Loop until successfully incremented the counter */
83 for (;;)
84 {
85 /* Make sure a rundown is not active */
86 if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
87
88 /* Add references */
89 NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;
90
91 /* Change the value */
92 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
93 if (NewValue == Value) return TRUE;
94
95 /* Update the value */
96 Value = NewValue;
97 }
98 }
99
100 /*++
101 * @name ExfInitializeRundownProtection
102 * @implemented NT5.1
103 *
104 * The ExfInitializeRundownProtection routine initializes a rundown
105 * protection descriptor.
106 *
107 * @param RunRef
108 * Pointer to a rundown reference descriptor.
109 *
110 * @return None.
111 *
112 * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
113 *
114 *--*/
115 VOID
116 FASTCALL
117 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
118 {
119 /* Set the count to zero */
120 RunRef->Count = 0;
121 }
122
123 /*++
124 * @name ExfReInitializeRundownProtection
125 * @implemented NT5.1
126 *
127 * The ExfReInitializeRundownProtection routine re-initializes a rundown
128 * protection descriptor.
129 *
130 * @param RunRef
131 * Pointer to a rundown reference descriptor.
132 *
133 * @return None.
134 *
135 * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
136 *
137 *--*/
138 VOID
139 FASTCALL
140 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
141 {
142 PAGED_CODE();
143
144 /* Sanity check */
145 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
146
147 /* Reset the count */
148 ExpSetRundown(RunRef, 0);
149 }
150
151 /*++
152 * @name ExfRundownCompleted
153 * @implemented NT5.1
154 *
155 * The ExfRundownCompleted routine completes the rundown of the specified
156 * descriptor by setting the active bit.
157 *
158 * @param RunRef
159 * Pointer to a rundown reference descriptor.
160 *
161 * @return None.
162 *
163 * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
164 *
165 *--*/
166 VOID
167 FASTCALL
168 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
169 {
170 PAGED_CODE();
171
172 /* Sanity check */
173 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
174
175 /* Mark the counter as active */
176 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
177 }
178
179 /*++
180 * @name ExfReleaseRundownProtection
181 * @implemented NT5.1
182 *
183 * The ExfReleaseRundownProtection routine releases the rundown protection
184 * reference for the specified descriptor.
185 *
186 * @param RunRef
187 * Pointer to a rundown reference descriptor.
188 *
189 * @return None.
190 *
191 * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
192 *
193 *--*/
194 VOID
195 FASTCALL
196 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
197 {
198 ULONG_PTR Value = RunRef->Count, NewValue;
199 PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
200
201 /* Loop until successfully incremented the counter */
202 for (;;)
203 {
204 /* Check if rundown is not active */
205 if (!(Value & EX_RUNDOWN_ACTIVE))
206 {
207 /* Sanity check */
208 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
209
210 /* Get the new value */
211 NewValue = Value - EX_RUNDOWN_COUNT_INC;
212
213 /* Change the value */
214 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
215 if (NewValue == Value) break;
216
217 /* Update value */
218 Value = NewValue;
219 }
220 else
221 {
222 /* Get the wait block */
223 WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
224 ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
225
226 /* Remove the one count */
227 if (!InterlockedDecrementSizeT(&WaitBlock->Count))
228 {
229 /* We're down to 0 now, so signal the event */
230 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
231 }
232
233 /* We're all done */
234 break;
235 }
236 }
237 }
238
239 /*++
240 * @name ExfReleaseRundownProtectionEx
241 * @implemented NT5.2
242 *
243 * The ExfReleaseRundownProtectionEx routine releases multiple rundown
244 * protection references for the specified descriptor.
245 *
246 * @param RunRef
247 * Pointer to a rundown reference descriptor.
248 *
249 * @param Count
250 * Number of times to dereference the descriptor.
251 *
252 * @return None.
253 *
254 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
255 *
256 *--*/
257 VOID
258 FASTCALL
259 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
260 IN ULONG Count)
261 {
262 ULONG_PTR Value = RunRef->Count, NewValue;
263 PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
264
265 /* Loop until successfully incremented the counter */
266 for (;;)
267 {
268 /* Check if rundown is not active */
269 if (!(Value & EX_RUNDOWN_ACTIVE))
270 {
271 /* Sanity check */
272 ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) ||
273 (KeNumberProcessors > 1));
274
275 /* Get the new value */
276 NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;
277
278 /* Change the value */
279 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
280 if (NewValue == Value) break;
281
282 /* Update value */
283 Value = NewValue;
284 }
285 else
286 {
287 /* Get the wait block */
288 WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
289 ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
290
291 /* Remove the counts */
292 if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
293 -(LONG)Count) == (LONG)Count)
294 {
295 /* We're down to 0 now, so signal the event */
296 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
297 }
298
299 /* We're all done */
300 break;
301 }
302 }
303 }
304
305 /*++
306 * @name ExfWaitForRundownProtectionRelease
307 * @implemented NT5.1
308 *
309 * The ExfWaitForRundownProtectionRelease routine waits until the specified
310 * rundown descriptor has been released.
311 *
312 * @param RunRef
313 * Pointer to a rundown reference descriptor.
314 *
315 * @return None.
316 *
317 * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
318 * at IRQL <= APC_LEVEL.
319 *
320 *--*/
321 VOID
322 FASTCALL
323 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
324 {
325 ULONG_PTR Value, Count, NewValue;
326 EX_RUNDOWN_WAIT_BLOCK WaitBlock;
327 PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
328 PKEVENT Event;
329 PAGED_CODE();
330
331 /* Set the active bit */
332 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
333 if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
334
335 /* No event for now */
336 Event = NULL;
337 WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
338 EX_RUNDOWN_ACTIVE);
339
340 /* Start waitblock set loop */
341 for (;;)
342 {
343 /* Save the count */
344 Count = Value >> EX_RUNDOWN_COUNT_SHIFT;
345
346 /* If the count is over one and we don't have en event yet, create it */
347 if ((Count) && !(Event))
348 {
349 /* Initialize the event */
350 KeInitializeEvent(&WaitBlock.WakeEvent,
351 SynchronizationEvent,
352 FALSE);
353
354 /* Set the pointer */
355 Event = &WaitBlock.WakeEvent;
356 }
357
358 /* Set the count */
359 WaitBlock.Count = Count;
360
361 /* Now set the pointer */
362 NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
363 if (NewValue == Value) break;
364
365 /* Loop again */
366 Value = NewValue;
367 ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
368 }
369
370 /* If the count was 0, we're done */
371 if (!Count) return;
372
373 /* Wait for whoever needs to release to notify us */
374 KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
375 ASSERT(WaitBlock.Count == 0);
376 }
377
378 /* FIXME: STUBS **************************************************************/
379
380 /*
381 * @implemented NT5.2
382 */
383 BOOLEAN
384 FASTCALL
385 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
386 {
387 PEX_RUNDOWN_REF RunRef;
388
389 RunRef = ExGetRunRefForCurrentProcessor(RunRefCacheAware);
390 return _ExAcquireRundownProtection(RunRef);
391 }
392
393 /*
394 * @unimplemented NT5.2
395 */
396 BOOLEAN
397 FASTCALL
398 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
399 IN ULONG Count)
400 {
401 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
402 DBG_UNREFERENCED_PARAMETER(Count);
403 UNIMPLEMENTED;
404 return FALSE;
405 }
406
407 /*
408 * @implemented NT5.2
409 */
410 VOID
411 FASTCALL
412 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
413 {
414 PEX_RUNDOWN_REF RunRef;
415
416 RunRef = ExGetRunRefForCurrentProcessor(RunRefCacheAware);
417 return _ExReleaseRundownProtection(RunRef);
418 }
419
420 /*
421 * @unimplemented NT5.2
422 */
423 VOID
424 FASTCALL
425 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
426 IN ULONG Count)
427 {
428 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
429 DBG_UNREFERENCED_PARAMETER(Count);
430 UNIMPLEMENTED;
431 }
432
433 /*
434 * @unimplemented NT5.2
435 */
436 VOID
437 FASTCALL
438 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
439 {
440 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
441 UNIMPLEMENTED;
442 }
443
444 /*
445 * @unimplemented NT5.2
446 */
447 VOID
448 FASTCALL
449 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
450 {
451 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
452 UNIMPLEMENTED;
453 }
454
455 /*
456 * @unimplemented NT5.2
457 */
458 VOID
459 FASTCALL
460 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
461 {
462 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
463 UNIMPLEMENTED;
464 }
465
466 /*
467 * @implemented NT5.2
468 */
469 PEX_RUNDOWN_REF_CACHE_AWARE
470 NTAPI
471 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType,
472 IN ULONG Tag)
473 {
474 PVOID PoolToFree;
475 PEX_RUNDOWN_REF RunRef;
476 ULONG RunRefSize, Count, Offset;
477 PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;
478
479 PAGED_CODE();
480
481 /* Allocate the master structure */
482 RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
483 if (RunRefCacheAware == NULL)
484 {
485 return NULL;
486 }
487
488 /* Compute the size of each runref */
489 RunRefCacheAware->Number = KeNumberProcessors;
490 if (KeNumberProcessors <= 1)
491 {
492 RunRefSize = sizeof(EX_RUNDOWN_REF);
493 }
494 else
495 {
496 RunRefSize = KeGetRecommendedSharedDataAlignment();
497 ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
498 }
499
500 /* It must at least hold a EX_RUNDOWN_REF structure */
501 ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
502 RunRefCacheAware->RunRefSize = RunRefSize;
503
504 /* Allocate our runref pool */
505 PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
506 if (PoolToFree == NULL)
507 {
508 ExFreePoolWithTag(RunRefCacheAware, Tag);
509 return NULL;
510 }
511
512 /* On SMP, check for alignment */
513 if (RunRefCacheAware->Number > 1)
514 {
515 /* FIXME: properly align run refs */
516 UNIMPLEMENTED;
517 }
518
519 RunRefCacheAware->RunRefs = PoolToFree;
520 RunRefCacheAware->PoolToFree = PoolToFree;
521
522 /* And initialize runref */
523 if (RunRefCacheAware->Number != 0)
524 {
525 for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
526 {
527 Offset = RunRefCacheAware->RunRefSize * Count;
528 RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
529 RunRef->Count = 0;
530 }
531 }
532
533 return RunRefCacheAware;
534 }
535
536 /*
537 * @implemented NT5.2
538 */
539 VOID
540 NTAPI
541 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
542 {
543 PAGED_CODE();
544
545 /*
546 * This is to be called for RunRefCacheAware that were allocated with
547 * ExAllocateCacheAwareRundownProtection and not for user-allocated
548 * ones
549 */
550 ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);
551
552 /* We don't know the tag that as used for allocation */
553 ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
554 ExFreePoolWithTag(RunRefCacheAware, 0);
555 }
556
557 /*
558 * @implemented NT5.2
559 */
560 VOID
561 NTAPI
562 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
563 IN SIZE_T Size)
564 {
565 PVOID Pool;
566 PEX_RUNDOWN_REF RunRef;
567 ULONG Count, RunRefSize, Offset;
568
569 PAGED_CODE();
570
571 /* Get the user allocate pool for runrefs */
572 Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));
573
574 /* By default a runref is structure-sized */
575 RunRefSize = sizeof(EX_RUNDOWN_REF);
576
577 /*
578 * If we just have enough room for a single runref, deduce were on a single
579 * processor machine
580 */
581 if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
582 {
583 Count = 1;
584 }
585 else
586 {
587 /* FIXME: Properly align on SMP */
588 UNIMPLEMENTED;
589 }
590
591 /* Initialize the structure */
592 RunRefCacheAware->RunRefs = Pool;
593 RunRefCacheAware->RunRefSize = RunRefSize;
594 RunRefCacheAware->Number = Count;
595
596 /* There is no allocated pool! */
597 RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;
598
599 /* Initialize runref */
600 if (RunRefCacheAware->Number != 0)
601 {
602 for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
603 {
604 Offset = RunRefCacheAware->RunRefSize * Count;
605 RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
606 RunRef->Count = 0;
607 }
608 }
609 }
610
611 /*
612 * @implemented NT5.2
613 */
614 SIZE_T
615 NTAPI
616 ExSizeOfRundownProtectionCacheAware(VOID)
617 {
618 SIZE_T Size;
619
620 PAGED_CODE();
621
622 /* Compute the needed size for runrefs */
623 if (KeNumberProcessors <= 1)
624 {
625 Size = sizeof(EX_RUNDOWN_REF);
626 }
627 else
628 {
629 /* We +1, to have enough room for alignment */
630 Size = (KeNumberProcessors + 1) * KeGetRecommendedSharedDataAlignment();
631 }
632
633 /* Return total size (master structure and runrefs) */
634 return Size + sizeof(EX_RUNDOWN_REF_CACHE_AWARE);
635 }
636