[NTOSKRNL] Implement ExInitializeRundownProtectionCacheAware()
[reactos.git] / ntoskrnl / ex / rundown.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/ex/rundown.c
5 * PURPOSE: Rundown and Cache-Aware Rundown Protection
6 * PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
7 * Thomas Weidenmueller
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 /* FUNCTIONS *****************************************************************/
17
18 /*++
19 * @name ExfAcquireRundownProtection
20 * @implemented NT5.1
21 *
22 * The ExfAcquireRundownProtection routine acquires rundown protection for
23 * the specified descriptor.
24 *
25 * @param RunRef
26 * Pointer to a rundown reference descriptor.
27 *
28 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
29 *
30 * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
31 *
32 *--*/
33 BOOLEAN
34 FASTCALL
35 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
36 {
37 ULONG_PTR Value = RunRef->Count, NewValue;
38
39 /* Loop until successfully incremented the counter */
40 for (;;)
41 {
42 /* Make sure a rundown is not active */
43 if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
44
45 /* Add a reference */
46 NewValue = Value + EX_RUNDOWN_COUNT_INC;
47
48 /* Change the value */
49 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
50 if (NewValue == Value) return TRUE;
51
52 /* Update it */
53 Value = NewValue;
54 }
55 }
56
57 /*++
58 * @name ExfAcquireRundownProtectionEx
59 * @implemented NT5.2
60 *
61 * The ExfAcquireRundownProtectionEx routine acquires multiple rundown
62 * protection references for the specified descriptor.
63 *
64 * @param RunRef
65 * Pointer to a rundown reference descriptor.
66 *
67 * @param Count
68 * Number of times to reference the descriptor.
69 *
70 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
71 *
72 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
73 *
74 *--*/
75 BOOLEAN
76 FASTCALL
77 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
78 IN ULONG Count)
79 {
80 ULONG_PTR Value = RunRef->Count, NewValue;
81
82 /* Loop until successfully incremented the counter */
83 for (;;)
84 {
85 /* Make sure a rundown is not active */
86 if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
87
88 /* Add references */
89 NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;
90
91 /* Change the value */
92 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
93 if (NewValue == Value) return TRUE;
94
95 /* Update the value */
96 Value = NewValue;
97 }
98 }
99
100 /*++
101 * @name ExfInitializeRundownProtection
102 * @implemented NT5.1
103 *
104 * The ExfInitializeRundownProtection routine initializes a rundown
105 * protection descriptor.
106 *
107 * @param RunRef
108 * Pointer to a rundown reference descriptor.
109 *
110 * @return None.
111 *
112 * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
113 *
114 *--*/
115 VOID
116 FASTCALL
117 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
118 {
119 /* Set the count to zero */
120 RunRef->Count = 0;
121 }
122
123 /*++
124 * @name ExfReInitializeRundownProtection
125 * @implemented NT5.1
126 *
127 * The ExfReInitializeRundownProtection routine re-initializes a rundown
128 * protection descriptor.
129 *
130 * @param RunRef
131 * Pointer to a rundown reference descriptor.
132 *
133 * @return None.
134 *
135 * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
136 *
137 *--*/
138 VOID
139 FASTCALL
140 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
141 {
142 PAGED_CODE();
143
144 /* Sanity check */
145 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
146
147 /* Reset the count */
148 ExpSetRundown(RunRef, 0);
149 }
150
151 /*++
152 * @name ExfRundownCompleted
153 * @implemented NT5.1
154 *
155 * The ExfRundownCompleted routine completes the rundown of the specified
156 * descriptor by setting the active bit.
157 *
158 * @param RunRef
159 * Pointer to a rundown reference descriptor.
160 *
161 * @return None.
162 *
163 * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
164 *
165 *--*/
166 VOID
167 FASTCALL
168 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
169 {
170 PAGED_CODE();
171
172 /* Sanity check */
173 ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
174
175 /* Mark the counter as active */
176 ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
177 }
178
179 /*++
180 * @name ExfReleaseRundownProtection
181 * @implemented NT5.1
182 *
183 * The ExfReleaseRundownProtection routine releases the rundown protection
184 * reference for the specified descriptor.
185 *
186 * @param RunRef
187 * Pointer to a rundown reference descriptor.
188 *
189 * @return None.
190 *
191 * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
192 *
193 *--*/
194 VOID
195 FASTCALL
196 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
197 {
198 ULONG_PTR Value = RunRef->Count, NewValue;
199 PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
200
201 /* Loop until successfully incremented the counter */
202 for (;;)
203 {
204 /* Check if rundown is not active */
205 if (!(Value & EX_RUNDOWN_ACTIVE))
206 {
207 /* Sanity check */
208 ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
209
210 /* Get the new value */
211 NewValue = Value - EX_RUNDOWN_COUNT_INC;
212
213 /* Change the value */
214 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
215 if (NewValue == Value) break;
216
217 /* Update value */
218 Value = NewValue;
219 }
220 else
221 {
222 /* Get the wait block */
223 WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
224 ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
225
226 /* Remove the one count */
227 if (!InterlockedDecrementSizeT(&WaitBlock->Count))
228 {
229 /* We're down to 0 now, so signal the event */
230 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
231 }
232
233 /* We're all done */
234 break;
235 }
236 }
237 }
238
239 /*++
240 * @name ExfReleaseRundownProtectionEx
241 * @implemented NT5.2
242 *
243 * The ExfReleaseRundownProtectionEx routine releases multiple rundown
244 * protection references for the specified descriptor.
245 *
246 * @param RunRef
247 * Pointer to a rundown reference descriptor.
248 *
249 * @param Count
250 * Number of times to dereference the descriptor.
251 *
252 * @return None.
253 *
254 * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
255 *
256 *--*/
257 VOID
258 FASTCALL
259 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
260 IN ULONG Count)
261 {
262 ULONG_PTR Value = RunRef->Count, NewValue;
263 PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
264
265 /* Loop until successfully incremented the counter */
266 for (;;)
267 {
268 /* Check if rundown is not active */
269 if (!(Value & EX_RUNDOWN_ACTIVE))
270 {
271 /* Sanity check */
272 ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) ||
273 (KeNumberProcessors > 1));
274
275 /* Get the new value */
276 NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;
277
278 /* Change the value */
279 NewValue = ExpChangeRundown(RunRef, NewValue, Value);
280 if (NewValue == Value) break;
281
282 /* Update value */
283 Value = NewValue;
284 }
285 else
286 {
287 /* Get the wait block */
288 WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
289 ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
290
291 /* Remove the counts */
292 if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
293 -(LONG)Count) == (LONG)Count)
294 {
295 /* We're down to 0 now, so signal the event */
296 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
297 }
298
299 /* We're all done */
300 break;
301 }
302 }
303 }
304
305 /*++
306 * @name ExfWaitForRundownProtectionRelease
307 * @implemented NT5.1
308 *
309 * The ExfWaitForRundownProtectionRelease routine waits until the specified
310 * rundown descriptor has been released.
311 *
312 * @param RunRef
313 * Pointer to a rundown reference descriptor.
314 *
315 * @return None.
316 *
317 * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
318 * at IRQL <= APC_LEVEL.
319 *
320 *--*/
321 VOID
322 FASTCALL
323 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
324 {
325 ULONG_PTR Value, Count, NewValue;
326 EX_RUNDOWN_WAIT_BLOCK WaitBlock;
327 PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
328 PKEVENT Event;
329 PAGED_CODE();
330
331 /* Set the active bit */
332 Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
333 if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
334
335 /* No event for now */
336 Event = NULL;
337 WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
338 EX_RUNDOWN_ACTIVE);
339
340 /* Start waitblock set loop */
341 for (;;)
342 {
343 /* Save the count */
344 Count = Value >> EX_RUNDOWN_COUNT_SHIFT;
345
346 /* If the count is over one and we don't have en event yet, create it */
347 if ((Count) && !(Event))
348 {
349 /* Initialize the event */
350 KeInitializeEvent(&WaitBlock.WakeEvent,
351 SynchronizationEvent,
352 FALSE);
353
354 /* Set the pointer */
355 Event = &WaitBlock.WakeEvent;
356 }
357
358 /* Set the count */
359 WaitBlock.Count = Count;
360
361 /* Now set the pointer */
362 NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
363 if (NewValue == Value) break;
364
365 /* Loop again */
366 Value = NewValue;
367 ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
368 }
369
370 /* If the count was 0, we're done */
371 if (!Count) return;
372
373 /* Wait for whoever needs to release to notify us */
374 KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
375 ASSERT(WaitBlock.Count == 0);
376 }
377
378 /* FIXME: STUBS **************************************************************/
379
380 /*
381 * @unimplemented NT5.2
382 */
383 BOOLEAN
384 FASTCALL
385 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
386 {
387 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
388 UNIMPLEMENTED;
389 return FALSE;
390 }
391
392 /*
393 * @unimplemented NT5.2
394 */
395 BOOLEAN
396 FASTCALL
397 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
398 IN ULONG Count)
399 {
400 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
401 DBG_UNREFERENCED_PARAMETER(Count);
402 UNIMPLEMENTED;
403 return FALSE;
404 }
405
406 /*
407 * @unimplemented NT5.2
408 */
409 VOID
410 FASTCALL
411 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
412 {
413 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
414 UNIMPLEMENTED;
415 }
416
417 /*
418 * @unimplemented NT5.2
419 */
420 VOID
421 FASTCALL
422 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
423 IN ULONG Count)
424 {
425 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
426 DBG_UNREFERENCED_PARAMETER(Count);
427 UNIMPLEMENTED;
428 }
429
430 /*
431 * @unimplemented NT5.2
432 */
433 VOID
434 FASTCALL
435 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
436 {
437 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
438 UNIMPLEMENTED;
439 }
440
441 /*
442 * @unimplemented NT5.2
443 */
444 VOID
445 FASTCALL
446 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
447 {
448 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
449 UNIMPLEMENTED;
450 }
451
452 /*
453 * @unimplemented NT5.2
454 */
455 VOID
456 FASTCALL
457 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
458 {
459 DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
460 UNIMPLEMENTED;
461 }
462
463 /*
464 * @implemented NT5.2
465 */
466 PEX_RUNDOWN_REF_CACHE_AWARE
467 NTAPI
468 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType,
469 IN ULONG Tag)
470 {
471 PVOID PoolToFree;
472 PEX_RUNDOWN_REF RunRef;
473 ULONG RunRefSize, Count, Offset;
474 PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;
475
476 PAGED_CODE();
477
478 /* Allocate the master structure */
479 RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
480 if (RunRefCacheAware == NULL)
481 {
482 return NULL;
483 }
484
485 /* Compute the size of each runref */
486 RunRefCacheAware->Number = KeNumberProcessors;
487 if (KeNumberProcessors <= 1)
488 {
489 RunRefSize = sizeof(EX_RUNDOWN_REF);
490 }
491 else
492 {
493 RunRefSize = KeGetRecommendedSharedDataAlignment();
494 ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
495 }
496
497 /* It must at least hold a EX_RUNDOWN_REF structure */
498 ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
499 RunRefCacheAware->RunRefSize = RunRefSize;
500
501 /* Allocate our runref pool */
502 PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
503 if (PoolToFree == NULL)
504 {
505 ExFreePoolWithTag(RunRefCacheAware, Tag);
506 return NULL;
507 }
508
509 /* On SMP, check for alignment */
510 if (RunRefCacheAware->Number > 1)
511 {
512 /* FIXME: properly align run refs */
513 UNIMPLEMENTED;
514 }
515
516 RunRefCacheAware->RunRefs = PoolToFree;
517 RunRefCacheAware->PoolToFree = PoolToFree;
518
519 /* And initialize runref */
520 if (RunRefCacheAware->Number != 0)
521 {
522 for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
523 {
524 Offset = RunRefCacheAware->RunRefSize * Count;
525 RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
526 RunRef->Count = 0;
527 }
528 }
529
530 return RunRefCacheAware;
531 }
532
533 /*
534 * @implemented NT5.2
535 */
536 VOID
537 NTAPI
538 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
539 {
540 PAGED_CODE();
541
542 /*
543 * This is to be called for RunRefCacheAware that were allocated with
544 * ExAllocateCacheAwareRundownProtection and not for user-allocated
545 * ones
546 */
547 ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);
548
549 /* We don't know the tag that as used for allocation */
550 ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
551 ExFreePoolWithTag(RunRefCacheAware, 0);
552 }
553
554 /*
555 * @implemented NT5.2
556 */
557 VOID
558 NTAPI
559 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
560 IN SIZE_T Size)
561 {
562 PVOID Pool;
563 PEX_RUNDOWN_REF RunRef;
564 ULONG Count, RunRefSize, Offset;
565
566 PAGED_CODE();
567
568 /* Get the user allocate pool for runrefs */
569 Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));
570
571 /* By default a runref is structure-sized */
572 RunRefSize = sizeof(EX_RUNDOWN_REF);
573
574 /*
575 * If we just have enough room for a single runref, deduce were on a single
576 * processor machine
577 */
578 if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
579 {
580 Count = 1;
581 }
582 else
583 {
584 /* FIXME: Properly align on SMP */
585 UNIMPLEMENTED;
586 }
587
588 /* Initialize the structure */
589 RunRefCacheAware->RunRefs = Pool;
590 RunRefCacheAware->RunRefSize = RunRefSize;
591 RunRefCacheAware->Number = Count;
592
593 /* There is no allocated pool! */
594 RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;
595
596 /* Initialize runref */
597 if (RunRefCacheAware->Number != 0)
598 {
599 for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
600 {
601 Offset = RunRefCacheAware->RunRefSize * Count;
602 RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
603 RunRef->Count = 0;
604 }
605 }
606 }
607
608 /*
609 * @unimplemented NT5.2
610 */
611 SIZE_T
612 NTAPI
613 ExSizeOfRundownProtectionCacheAware(VOID)
614 {
615 UNIMPLEMENTED;
616 return 0;
617 }
618