ntoskrnl/ex/rundown.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS Kernel
   4  * FILE:            ntoskrnl/ex/rundown.c
   5  * PURPOSE:         Rundown and Cache-Aware Rundown Protection
   6  * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
   7  *                  Thomas Weidenmueller
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 /* FUNCTIONS *****************************************************************/
  17
  18 /*++
  19  * @name ExfAcquireRundownProtection
  20  * @implemented NT5.1
  21  *
  22  *     The ExfAcquireRundownProtection routine acquires rundown protection for
  23  *     the specified descriptor.
  24  *
  25  * @param RunRef
  26  *        Pointer to a rundown reference descriptor.
  27  *
  28  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
  29  *
  30  * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
  31  *
  32  *--*/
  33 BOOLEAN
  34 FASTCALL
  35 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
  36 {
  37     ULONG_PTR Value = RunRef->Count, NewValue;
  38
  39     /* Loop until successfully incremented the counter */
  40     for (;;)
  41     {
  42         /* Make sure a rundown is not active */
  43         if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
  44
  45         /* Add a reference */
  46         NewValue = Value + EX_RUNDOWN_COUNT_INC;
  47
  48         /* Change the value */
  49         NewValue = ExpChangeRundown(RunRef, NewValue, Value);
  50         if (NewValue == Value) return TRUE;
  51
  52         /* Update it */
  53         Value = NewValue;
  54     }
  55 }
  56
  57 /*++
  58  * @name ExfAcquireRundownProtectionEx
  59  * @implemented NT5.2
  60  *
  61  *     The ExfAcquireRundownProtectionEx routine acquires multiple rundown
  62  *     protection references for the specified descriptor.
  63  *
  64  * @param RunRef
  65  *        Pointer to a rundown reference descriptor.
  66  *
  67  * @param Count
  68  *         Number of times to reference the descriptor.
  69  *
  70  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
  71  *
  72  * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
  73  *
  74  *--*/
  75 BOOLEAN
  76 FASTCALL
  77 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
  78                               IN ULONG Count)
  79 {
  80     ULONG_PTR Value = RunRef->Count, NewValue;
  81
  82     /* Loop until successfully incremented the counter */
  83     for (;;)
  84     {
  85         /* Make sure a rundown is not active */
  86         if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
  87
  88         /* Add references */
  89         NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;
  90
  91         /* Change the value */
  92         NewValue = ExpChangeRundown(RunRef, NewValue, Value);
  93         if (NewValue == Value) return TRUE;
  94
  95         /* Update the value */
  96         Value = NewValue;
  97     }
  98 }
  99
 100 /*++
 101  * @name ExfInitializeRundownProtection
 102  * @implemented NT5.1
 103  *
 104  *     The ExfInitializeRundownProtection routine initializes a rundown
 105  *     protection descriptor.
 106  *
 107  * @param RunRef
 108  *        Pointer to a rundown reference descriptor.
 109  *
 110  * @return None.
 111  *
 112  * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
 113  *
 114  *--*/
 115 VOID
 116 FASTCALL
 117 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 118 {
 119     /* Set the count to zero */
 120     RunRef->Count = 0;
 121 }
 122
 123 /*++
 124  * @name ExfReInitializeRundownProtection
 125  * @implemented NT5.1
 126  *
 127  *     The ExfReInitializeRundownProtection routine re-initializes a rundown
 128  *     protection descriptor.
 129  *
 130  * @param RunRef
 131  *        Pointer to a rundown reference descriptor.
 132  *
 133  * @return None.
 134  *
 135  * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
 136  *
 137  *--*/
 138 VOID
 139 FASTCALL
 140 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 141 {
 142     PAGED_CODE();
 143
 144     /* Sanity check */
 145     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 146
 147     /* Reset the count */
 148     ExpSetRundown(RunRef, 0);
 149 }
 150
 151 /*++
 152  * @name ExfRundownCompleted
 153  * @implemented NT5.1
 154  *
 155  *     The ExfRundownCompleted routine completes the rundown of the specified
 156  *     descriptor by setting the active bit.
 157  *
 158  * @param RunRef
 159  *        Pointer to a rundown reference descriptor.
 160  *
 161  * @return None.
 162  *
 163  * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
 164  *
 165  *--*/
 166 VOID
 167 FASTCALL
 168 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 169 {
 170     PAGED_CODE();
 171
 172     /* Sanity check */
 173     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 174
 175     /* Mark the counter as active */
 176     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 177 }
 178
 179 /*++
 180  * @name ExfReleaseRundownProtection
 181  * @implemented NT5.1
 182  *
 183  *     The ExfReleaseRundownProtection routine releases the rundown protection
 184  *     reference for the specified descriptor.
 185  *
 186  * @param RunRef
 187  *        Pointer to a rundown reference descriptor.
 188  *
 189  * @return None.
 190  *
 191  * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
 192  *
 193  *--*/
 194 VOID
 195 FASTCALL
 196 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 197 {
 198     ULONG_PTR Value = RunRef->Count, NewValue;
 199     PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
 200
 201     /* Loop until successfully incremented the counter */
 202     for (;;)
 203     {
 204         /* Check if rundown is not active */
 205         if (!(Value & EX_RUNDOWN_ACTIVE))
 206         {
 207             /* Sanity check */
 208             ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 209
 210             /* Get the new value */
 211             NewValue = Value - EX_RUNDOWN_COUNT_INC;
 212
 213             /* Change the value */
 214             NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 215             if (NewValue == Value) break;
 216
 217             /* Update value */
 218             Value = NewValue;
 219         }
 220         else
 221         {
 222             /* Get the wait block */
 223             WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
 224             ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
 225
 226             /* Remove the one count */
 227             if (!InterlockedDecrementSizeT(&WaitBlock->Count))
 228             {
 229                 /* We're down to 0 now, so signal the event */
 230                 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
 231             }
 232
 233             /* We're all done */
 234             break;
 235         }
 236     }
 237 }
 238
 239 /*++
 240  * @name ExfReleaseRundownProtectionEx
 241  * @implemented NT5.2
 242  *
 243  *     The ExfReleaseRundownProtectionEx routine releases multiple rundown
 244  *     protection references for the specified descriptor.
 245  *
 246  * @param RunRef
 247  *        Pointer to a rundown reference descriptor.
 248  *
 249  * @param Count
 250  *         Number of times to dereference the descriptor.
 251  *
 252  * @return None.
 253  *
 254  * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
 255  *
 256  *--*/
 257 VOID
 258 FASTCALL
 259 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
 260                               IN ULONG Count)
 261 {
 262     ULONG_PTR Value = RunRef->Count, NewValue;
 263     PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
 264
 265     /* Loop until successfully incremented the counter */
 266     for (;;)
 267     {
 268         /* Check if rundown is not active */
 269         if (!(Value & EX_RUNDOWN_ACTIVE))
 270         {
 271             /* Sanity check */
 272             ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) ||
 273                    (KeNumberProcessors > 1));
 274
 275             /* Get the new value */
 276             NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;
 277
 278             /* Change the value */
 279             NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 280             if (NewValue == Value) break;
 281
 282             /* Update value */
 283             Value = NewValue;
 284         }
 285         else
 286         {
 287             /* Get the wait block */
 288             WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
 289             ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
 290
 291             /* Remove the counts */
 292             if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
 293                                             -(LONG)Count) == (LONG)Count)
 294             {
 295                 /* We're down to 0 now, so signal the event */
 296                 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
 297             }
 298
 299             /* We're all done */
 300             break;
 301         }
 302     }
 303 }
 304
 305 /*++
 306  * @name ExfWaitForRundownProtectionRelease
 307  * @implemented NT5.1
 308  *
 309  *     The ExfWaitForRundownProtectionRelease routine waits until the specified
 310  *     rundown descriptor has been released.
 311  *
 312  * @param RunRef
 313  *        Pointer to a rundown reference descriptor.
 314  *
 315  * @return None.
 316  *
 317  * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
 318  *          at IRQL <= APC_LEVEL.
 319  *
 320  *--*/
 321 VOID
 322 FASTCALL
 323 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 324 {
 325     ULONG_PTR Value, Count, NewValue;
 326     EX_RUNDOWN_WAIT_BLOCK WaitBlock;
 327     PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
 328     PKEVENT Event;
 329     PAGED_CODE();
 330
 331     /* Set the active bit */
 332     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 333     if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
 334
 335     /* No event for now */
 336     Event = NULL;
 337     WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
 338                                                 EX_RUNDOWN_ACTIVE);
 339
 340     /* Start waitblock set loop */
 341     for (;;)
 342     {
 343         /* Save the count */
 344         Count = Value >> EX_RUNDOWN_COUNT_SHIFT;
 345
 346         /* If the count is over one and we don't have en event yet, create it */
 347         if ((Count) && !(Event))
 348         {
 349             /* Initialize the event */
 350             KeInitializeEvent(&WaitBlock.WakeEvent,
 351                               SynchronizationEvent,
 352                               FALSE);
 353
 354             /* Set the pointer */
 355             Event = &WaitBlock.WakeEvent;
 356         }
 357
 358         /* Set the count */
 359         WaitBlock.Count = Count;
 360
 361         /* Now set the pointer */
 362         NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
 363         if (NewValue == Value) break;
 364
 365         /* Loop again */
 366         Value = NewValue;
 367         ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
 368     }
 369
 370     /* If the count was 0, we're done */
 371     if (!Count) return;
 372
 373     /* Wait for whoever needs to release to notify us */
 374     KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
 375     ASSERT(WaitBlock.Count == 0);
 376 }
 377
 378 /* FIXME: STUBS **************************************************************/
 379
 380 /*
 381  * @unimplemented NT5.2
 382  */
 383 BOOLEAN
 384 FASTCALL
 385 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 386 {
 387     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 388     UNIMPLEMENTED;
 389     return FALSE;
 390 }
 391
 392 /*
 393  * @unimplemented NT5.2
 394  */
 395 BOOLEAN
 396 FASTCALL
 397 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 398                                         IN ULONG Count)
 399 {
 400     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 401     DBG_UNREFERENCED_PARAMETER(Count);
 402     UNIMPLEMENTED;
 403     return FALSE;
 404 }
 405
 406 /*
 407  * @unimplemented NT5.2
 408  */
 409 VOID
 410 FASTCALL
 411 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 412 {
 413     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 414     UNIMPLEMENTED;
 415 }
 416
 417 /*
 418  * @unimplemented NT5.2
 419  */
 420 VOID
 421 FASTCALL
 422 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 423                                         IN ULONG Count)
 424 {
 425     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 426     DBG_UNREFERENCED_PARAMETER(Count);
 427     UNIMPLEMENTED;
 428 }
 429
 430 /*
 431  * @unimplemented NT5.2
 432  */
 433 VOID
 434 FASTCALL
 435 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 436 {
 437     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 438     UNIMPLEMENTED;
 439 }
 440
 441 /*
 442  * @unimplemented NT5.2
 443  */
 444 VOID
 445 FASTCALL
 446 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 447 {
 448     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 449     UNIMPLEMENTED;
 450 }
 451
 452 /*
 453  * @unimplemented NT5.2
 454  */
 455 VOID
 456 FASTCALL
 457 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 458 {
 459     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 460     UNIMPLEMENTED;
 461 }
 462
 463 /*
 464  * @implemented NT5.2
 465  */
 466 PEX_RUNDOWN_REF_CACHE_AWARE
 467 NTAPI
 468 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType,
 469                                       IN ULONG Tag)
 470 {
 471     PVOID PoolToFree;
 472     PEX_RUNDOWN_REF RunRef;
 473     ULONG RunRefSize, Count, Offset;
 474     PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;
 475
 476     PAGED_CODE();
 477
 478     /* Allocate the master structure */
 479     RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
 480     if (RunRefCacheAware == NULL)
 481     {
 482         return NULL;
 483     }
 484
 485     /* Compute the size of each runref */
 486     RunRefCacheAware->Number = KeNumberProcessors;
 487     if (KeNumberProcessors <= 1)
 488     {
 489         RunRefSize = sizeof(EX_RUNDOWN_REF);
 490     }
 491     else
 492     {
 493         RunRefSize = KeGetRecommendedSharedDataAlignment();
 494         ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
 495     }
 496
 497     /* It must at least hold a EX_RUNDOWN_REF structure */
 498     ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
 499     RunRefCacheAware->RunRefSize = RunRefSize;
 500
 501     /* Allocate our runref pool */
 502     PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
 503     if (PoolToFree == NULL)
 504     {
 505         ExFreePoolWithTag(RunRefCacheAware, Tag);
 506         return NULL;
 507     }
 508
 509     /* On SMP, check for alignment */
 510     if (RunRefCacheAware->Number > 1)
 511     {
 512         /* FIXME: properly align run refs */
 513         UNIMPLEMENTED;
 514     }
 515
 516     RunRefCacheAware->RunRefs = PoolToFree;
 517     RunRefCacheAware->PoolToFree = PoolToFree;
 518
 519     /* And initialize runref */
 520     if (RunRefCacheAware->Number != 0)
 521     {
 522         for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
 523         {
 524             Offset = RunRefCacheAware->RunRefSize * Count;
 525             RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
 526             RunRef->Count = 0;
 527         }
 528     }
 529
 530     return RunRefCacheAware;
 531 }
 532
 533 /*
 534  * @implemented NT5.2
 535  */
 536 VOID
 537 NTAPI
 538 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 539 {
 540     PAGED_CODE();
 541
 542     /*
 543      * This is to be called for RunRefCacheAware that were allocated with
 544      * ExAllocateCacheAwareRundownProtection and not for user-allocated
 545      * ones
 546      */
 547     ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);
 548
 549     /* We don't know the tag that as used for allocation */
 550     ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
 551     ExFreePoolWithTag(RunRefCacheAware, 0);
 552 }
 553
 554 /*
 555  * @implemented NT5.2
 556  */
 557 VOID
 558 NTAPI
 559 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 560                                         IN SIZE_T Size)
 561 {
 562     PVOID Pool;
 563     PEX_RUNDOWN_REF RunRef;
 564     ULONG Count, RunRefSize, Offset;
 565
 566     PAGED_CODE();
 567
 568     /* Get the user allocate pool for runrefs */
 569     Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));
 570
 571     /* By default a runref is structure-sized */
 572     RunRefSize = sizeof(EX_RUNDOWN_REF);
 573
 574     /*
 575      * If we just have enough room for a single runref, deduce were on a single
 576      * processor machine
 577      */
 578     if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
 579     {
 580         Count = 1;
 581     }
 582     else
 583     {
 584         /* FIXME: Properly align on SMP */
 585         UNIMPLEMENTED;
 586     }
 587
 588     /* Initialize the structure */
 589     RunRefCacheAware->RunRefs = Pool;
 590     RunRefCacheAware->RunRefSize = RunRefSize;
 591     RunRefCacheAware->Number = Count;
 592
 593     /* There is no allocated pool! */
 594     RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;
 595
 596     /* Initialize runref */
 597     if (RunRefCacheAware->Number != 0)
 598     {
 599         for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
 600         {
 601             Offset = RunRefCacheAware->RunRefSize * Count;
 602             RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
 603             RunRef->Count = 0;
 604         }
 605     }
 606 }
 607
 608 /*
 609  * @implemented NT5.2
 610  */
 611 SIZE_T
 612 NTAPI
 613 ExSizeOfRundownProtectionCacheAware(VOID)
 614 {
 615     SIZE_T Size;
 616
 617     PAGED_CODE();
 618
 619     /* Compute the needed size for runrefs */
 620     if (KeNumberProcessors <= 1)
 621     {
 622         Size = sizeof(EX_RUNDOWN_REF);
 623     }
 624     else
 625     {
 626         /* We +1, to have enough room for alignment */
 627         Size = (KeNumberProcessors + 1) * KeGetRecommendedSharedDataAlignment();
 628     }
 629
 630     /* Return total size (master structure and runrefs) */
 631     return Size + sizeof(EX_RUNDOWN_REF_CACHE_AWARE);
 632 }
 633