ntoskrnl/ex/rundown.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS Kernel
   4  * FILE:            ntoskrnl/ex/rundown.c
   5  * PURPOSE:         Rundown and Cache-Aware Rundown Protection
   6  * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
   7  *                  Thomas Weidenmueller
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 /* FUNCTIONS *****************************************************************/
  17
  18 /*++
  19  * @name ExfAcquireRundownProtection
  20  * @implemented NT5.1
  21  *
  22  *     The ExfAcquireRundownProtection routine acquires rundown protection for
  23  *     the specified descriptor.
  24  *
  25  * @param RunRef
  26  *        Pointer to a rundown reference descriptor.
  27  *
  28  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
  29  *
  30  * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
  31  *
  32  *--*/
  33 BOOLEAN
  34 FASTCALL
  35 ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
  36 {
  37     ULONG_PTR Value = RunRef->Count, NewValue;
  38
  39     /* Loop until successfully incremented the counter */
  40     for (;;)
  41     {
  42         /* Make sure a rundown is not active */
  43         if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
  44
  45         /* Add a reference */
  46         NewValue = Value + EX_RUNDOWN_COUNT_INC;
  47
  48         /* Change the value */
  49         NewValue = ExpChangeRundown(RunRef, NewValue, Value);
  50         if (NewValue == Value) return TRUE;
  51
  52         /* Update it */
  53         Value = NewValue;
  54     }
  55 }
  56
  57 /*++
  58  * @name ExfAcquireRundownProtectionEx
  59  * @implemented NT5.2
  60  *
  61  *     The ExfAcquireRundownProtectionEx routine acquires multiple rundown
  62  *     protection references for the specified descriptor.
  63  *
  64  * @param RunRef
  65  *        Pointer to a rundown reference descriptor.
  66  *
  67  * @param Count
  68  *         Number of times to reference the descriptor.
  69  *
  70  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
  71  *
  72  * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
  73  *
  74  *--*/
  75 BOOLEAN
  76 FASTCALL
  77 ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
  78                               IN ULONG Count)
  79 {
  80     ULONG_PTR Value = RunRef->Count, NewValue;
  81
  82     /* Loop until successfully incremented the counter */
  83     for (;;)
  84     {
  85         /* Make sure a rundown is not active */
  86         if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
  87
  88         /* Add references */
  89         NewValue = Value + EX_RUNDOWN_COUNT_INC * Count;
  90
  91         /* Change the value */
  92         NewValue = ExpChangeRundown(RunRef, NewValue, Value);
  93         if (NewValue == Value) return TRUE;
  94
  95         /* Update the value */
  96         Value = NewValue;
  97     }
  98 }
  99
 100 /*++
 101  * @name ExfInitializeRundownProtection
 102  * @implemented NT5.1
 103  *
 104  *     The ExfInitializeRundownProtection routine initializes a rundown
 105  *     protection descriptor.
 106  *
 107  * @param RunRef
 108  *        Pointer to a rundown reference descriptor.
 109  *
 110  * @return None.
 111  *
 112  * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
 113  *
 114  *--*/
 115 VOID
 116 FASTCALL
 117 ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 118 {
 119     /* Set the count to zero */
 120     RunRef->Count = 0;
 121 }
 122
 123 /*++
 124  * @name ExfReInitializeRundownProtection
 125  * @implemented NT5.1
 126  *
 127  *     The ExfReInitializeRundownProtection routine re-initializes a rundown
 128  *     protection descriptor.
 129  *
 130  * @param RunRef
 131  *        Pointer to a rundown reference descriptor.
 132  *
 133  * @return None.
 134  *
 135  * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
 136  *
 137  *--*/
 138 VOID
 139 FASTCALL
 140 ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 141 {
 142     PAGED_CODE();
 143
 144     /* Sanity check */
 145     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 146
 147     /* Reset the count */
 148     ExpSetRundown(RunRef, 0);
 149 }
 150
 151 /*++
 152  * @name ExfRundownCompleted
 153  * @implemented NT5.1
 154  *
 155  *     The ExfRundownCompleted routine completes the rundown of the specified
 156  *     descriptor by setting the active bit.
 157  *
 158  * @param RunRef
 159  *        Pointer to a rundown reference descriptor.
 160  *
 161  * @return None.
 162  *
 163  * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
 164  *
 165  *--*/
 166 VOID
 167 FASTCALL
 168 ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 169 {
 170     PAGED_CODE();
 171
 172     /* Sanity check */
 173     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 174
 175     /* Mark the counter as active */
 176     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 177 }
 178
 179 /*++
 180  * @name ExfReleaseRundownProtection
 181  * @implemented NT5.1
 182  *
 183  *     The ExfReleaseRundownProtection routine releases the rundown protection
 184  *     reference for the specified descriptor.
 185  *
 186  * @param RunRef
 187  *        Pointer to a rundown reference descriptor.
 188  *
 189  * @return None.
 190  *
 191  * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
 192  *
 193  *--*/
 194 VOID
 195 FASTCALL
 196 ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 197 {
 198     ULONG_PTR Value = RunRef->Count, NewValue;
 199     PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
 200
 201     /* Loop until successfully incremented the counter */
 202     for (;;)
 203     {
 204         /* Check if rundown is not active */
 205         if (!(Value & EX_RUNDOWN_ACTIVE))
 206         {
 207             /* Sanity check */
 208             ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 209
 210             /* Get the new value */
 211             NewValue = Value - EX_RUNDOWN_COUNT_INC;
 212
 213             /* Change the value */
 214             NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 215             if (NewValue == Value) break;
 216
 217             /* Update value */
 218             Value = NewValue;
 219         }
 220         else
 221         {
 222             /* Get the wait block */
 223             WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
 224             ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
 225
 226             /* Remove the one count */
 227             if (!InterlockedDecrementSizeT(&WaitBlock->Count))
 228             {
 229                 /* We're down to 0 now, so signal the event */
 230                 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
 231             }
 232
 233             /* We're all done */
 234             break;
 235         }
 236     }
 237 }
 238
 239 /*++
 240  * @name ExfReleaseRundownProtectionEx
 241  * @implemented NT5.2
 242  *
 243  *     The ExfReleaseRundownProtectionEx routine releases multiple rundown
 244  *     protection references for the specified descriptor.
 245  *
 246  * @param RunRef
 247  *        Pointer to a rundown reference descriptor.
 248  *
 249  * @param Count
 250  *         Number of times to dereference the descriptor.
 251  *
 252  * @return None.
 253  *
 254  * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
 255  *
 256  *--*/
 257 VOID
 258 FASTCALL
 259 ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
 260                               IN ULONG Count)
 261 {
 262     ULONG_PTR Value = RunRef->Count, NewValue;
 263     PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
 264
 265     /* Loop until successfully incremented the counter */
 266     for (;;)
 267     {
 268         /* Check if rundown is not active */
 269         if (!(Value & EX_RUNDOWN_ACTIVE))
 270         {
 271             /* Sanity check */
 272             ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) ||
 273                    (KeNumberProcessors > 1));
 274
 275             /* Get the new value */
 276             NewValue = Value - EX_RUNDOWN_COUNT_INC * Count;
 277
 278             /* Change the value */
 279             NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 280             if (NewValue == Value) break;
 281
 282             /* Update value */
 283             Value = NewValue;
 284         }
 285         else
 286         {
 287             /* Get the wait block */
 288             WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
 289             ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
 290
 291             /* Remove the counts */
 292             if (InterlockedExchangeAddSizeT(&WaitBlock->Count,
 293                                             -(LONG)Count) == (LONG)Count)
 294             {
 295                 /* We're down to 0 now, so signal the event */
 296                 KeSetEvent(&WaitBlock->WakeEvent, IO_NO_INCREMENT, FALSE);
 297             }
 298
 299             /* We're all done */
 300             break;
 301         }
 302     }
 303 }
 304
 305 /*++
 306  * @name ExfWaitForRundownProtectionRelease
 307  * @implemented NT5.1
 308  *
 309  *     The ExfWaitForRundownProtectionRelease routine waits until the specified
 310  *     rundown descriptor has been released.
 311  *
 312  * @param RunRef
 313  *        Pointer to a rundown reference descriptor.
 314  *
 315  * @return None.
 316  *
 317  * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
 318  *          at IRQL <= APC_LEVEL.
 319  *
 320  *--*/
 321 VOID
 322 FASTCALL
 323 ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 324 {
 325     ULONG_PTR Value, Count, NewValue;
 326     EX_RUNDOWN_WAIT_BLOCK WaitBlock;
 327     PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
 328     PKEVENT Event;
 329     PAGED_CODE();
 330
 331     /* Set the active bit */
 332     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 333     if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
 334
 335     /* No event for now */
 336     Event = NULL;
 337     WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
 338                                                 EX_RUNDOWN_ACTIVE);
 339
 340     /* Start waitblock set loop */
 341     for (;;)
 342     {
 343         /* Save the count */
 344         Count = Value >> EX_RUNDOWN_COUNT_SHIFT;
 345
 346         /* If the count is over one and we don't have en event yet, create it */
 347         if ((Count) && !(Event))
 348         {
 349             /* Initialize the event */
 350             KeInitializeEvent(&WaitBlock.WakeEvent,
 351                               SynchronizationEvent,
 352                               FALSE);
 353
 354             /* Set the pointer */
 355             Event = &WaitBlock.WakeEvent;
 356         }
 357
 358         /* Set the count */
 359         WaitBlock.Count = Count;
 360
 361         /* Now set the pointer */
 362         NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
 363         if (NewValue == Value) break;
 364
 365         /* Loop again */
 366         Value = NewValue;
 367         ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
 368     }
 369
 370     /* If the count was 0, we're done */
 371     if (!Count) return;
 372
 373     /* Wait for whoever needs to release to notify us */
 374     KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
 375     ASSERT(WaitBlock.Count == 0);
 376 }
 377
 378 /* FIXME: STUBS **************************************************************/
 379
 380 /*
 381  * @implemented NT5.2
 382  */
 383 BOOLEAN
 384 FASTCALL
 385 ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 386 {
 387     ULONG Value;
 388     BOOLEAN Acquired;
 389     PEX_RUNDOWN_REF RunRef;
 390
 391     RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 392                                RunRefCacheAware->RunRefSize *
 393                                (KeGetCurrentProcessorNumber() % RunRefCacheAware->Number));
 394
 395     /* Get current value */
 396     Value = RunRef->Count & !EX_RUNDOWN_ACTIVE;
 397     /* Try to acquire the quick way if already active */
 398     if (ExpChangeRundown(RunRef,
 399                          ((RunRef->Count & !EX_RUNDOWN_ACTIVE) + EX_RUNDOWN_COUNT_INC),
 400                          Value) == Value)
 401     {
 402         Acquired = 1;
 403     }
 404     else
 405     {
 406         Acquired = ExfAcquireRundownProtection(RunRef);
 407     }
 408
 409     return Acquired;
 410 }
 411
 412 /*
 413  * @unimplemented NT5.2
 414  */
 415 BOOLEAN
 416 FASTCALL
 417 ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 418                                         IN ULONG Count)
 419 {
 420     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 421     DBG_UNREFERENCED_PARAMETER(Count);
 422     UNIMPLEMENTED;
 423     return FALSE;
 424 }
 425
 426 /*
 427  * @implemented NT5.2
 428  */
 429 VOID
 430 FASTCALL
 431 ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 432 {
 433     ULONG Value;
 434     PEX_RUNDOWN_REF RunRef;
 435
 436     RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 437                                RunRefCacheAware->RunRefSize *
 438                                (KeGetCurrentProcessorNumber() % RunRefCacheAware->Number));
 439
 440     /* Get current value */
 441     Value = RunRef->Count & !EX_RUNDOWN_ACTIVE;
 442     /* Try to release the quick way if multiple actived */
 443     if (ExpChangeRundown(RunRef,
 444                          ((RunRef->Count & !EX_RUNDOWN_ACTIVE) - EX_RUNDOWN_COUNT_INC),
 445                          Value) == Value)
 446     {
 447         /* Sanity check */
 448         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 449     }
 450     else
 451     {
 452         ExfReleaseRundownProtection(RunRef);
 453     }
 454 }
 455
 456 /*
 457  * @unimplemented NT5.2
 458  */
 459 VOID
 460 FASTCALL
 461 ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 462                                         IN ULONG Count)
 463 {
 464     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 465     DBG_UNREFERENCED_PARAMETER(Count);
 466     UNIMPLEMENTED;
 467 }
 468
 469 /*
 470  * @unimplemented NT5.2
 471  */
 472 VOID
 473 FASTCALL
 474 ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 475 {
 476     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 477     UNIMPLEMENTED;
 478 }
 479
 480 /*
 481  * @unimplemented NT5.2
 482  */
 483 VOID
 484 FASTCALL
 485 ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 486 {
 487     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 488     UNIMPLEMENTED;
 489 }
 490
 491 /*
 492  * @unimplemented NT5.2
 493  */
 494 VOID
 495 FASTCALL
 496 ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 497 {
 498     DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
 499     UNIMPLEMENTED;
 500 }
 501
 502 /*
 503  * @implemented NT5.2
 504  */
 505 PEX_RUNDOWN_REF_CACHE_AWARE
 506 NTAPI
 507 ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType,
 508                                       IN ULONG Tag)
 509 {
 510     PVOID PoolToFree;
 511     PEX_RUNDOWN_REF RunRef;
 512     ULONG RunRefSize, Count, Offset;
 513     PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware;
 514
 515     PAGED_CODE();
 516
 517     /* Allocate the master structure */
 518     RunRefCacheAware = ExAllocatePoolWithTag(PoolType, sizeof(EX_RUNDOWN_REF_CACHE_AWARE), Tag);
 519     if (RunRefCacheAware == NULL)
 520     {
 521         return NULL;
 522     }
 523
 524     /* Compute the size of each runref */
 525     RunRefCacheAware->Number = KeNumberProcessors;
 526     if (KeNumberProcessors <= 1)
 527     {
 528         RunRefSize = sizeof(EX_RUNDOWN_REF);
 529     }
 530     else
 531     {
 532         RunRefSize = KeGetRecommendedSharedDataAlignment();
 533         ASSERT((RunRefSize & (RunRefSize - 1)) == 0);
 534     }
 535
 536     /* It must at least hold a EX_RUNDOWN_REF structure */
 537     ASSERT(sizeof(EX_RUNDOWN_REF) <= RunRefSize);
 538     RunRefCacheAware->RunRefSize = RunRefSize;
 539
 540     /* Allocate our runref pool */
 541     PoolToFree = ExAllocatePoolWithTag(PoolType, RunRefSize * RunRefCacheAware->Number, Tag);
 542     if (PoolToFree == NULL)
 543     {
 544         ExFreePoolWithTag(RunRefCacheAware, Tag);
 545         return NULL;
 546     }
 547
 548     /* On SMP, check for alignment */
 549     if (RunRefCacheAware->Number > 1)
 550     {
 551         /* FIXME: properly align run refs */
 552         UNIMPLEMENTED;
 553     }
 554
 555     RunRefCacheAware->RunRefs = PoolToFree;
 556     RunRefCacheAware->PoolToFree = PoolToFree;
 557
 558     /* And initialize runref */
 559     if (RunRefCacheAware->Number != 0)
 560     {
 561         for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
 562         {
 563             Offset = RunRefCacheAware->RunRefSize * Count;
 564             RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
 565             RunRef->Count = 0;
 566         }
 567     }
 568
 569     return RunRefCacheAware;
 570 }
 571
 572 /*
 573  * @implemented NT5.2
 574  */
 575 VOID
 576 NTAPI
 577 ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 578 {
 579     PAGED_CODE();
 580
 581     /*
 582      * This is to be called for RunRefCacheAware that were allocated with
 583      * ExAllocateCacheAwareRundownProtection and not for user-allocated
 584      * ones
 585      */
 586     ASSERT(RunRefCacheAware->PoolToFree != (PVOID)0xBADCA11);
 587
 588     /* We don't know the tag that as used for allocation */
 589     ExFreePoolWithTag(RunRefCacheAware->PoolToFree, 0);
 590     ExFreePoolWithTag(RunRefCacheAware, 0);
 591 }
 592
 593 /*
 594  * @implemented NT5.2
 595  */
 596 VOID
 597 NTAPI
 598 ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 599                                         IN SIZE_T Size)
 600 {
 601     PVOID Pool;
 602     PEX_RUNDOWN_REF RunRef;
 603     ULONG Count, RunRefSize, Offset;
 604
 605     PAGED_CODE();
 606
 607     /* Get the user allocate pool for runrefs */
 608     Pool = (PVOID)((ULONG_PTR)RunRefCacheAware + sizeof(EX_RUNDOWN_REF_CACHE_AWARE));
 609
 610     /* By default a runref is structure-sized */
 611     RunRefSize = sizeof(EX_RUNDOWN_REF);
 612
 613     /*
 614      * If we just have enough room for a single runref, deduce were on a single
 615      * processor machine
 616      */
 617     if (Size == sizeof(EX_RUNDOWN_REF_CACHE_AWARE) + sizeof(EX_RUNDOWN_REF))
 618     {
 619         Count = 1;
 620     }
 621     else
 622     {
 623         /* FIXME: Properly align on SMP */
 624         UNIMPLEMENTED;
 625     }
 626
 627     /* Initialize the structure */
 628     RunRefCacheAware->RunRefs = Pool;
 629     RunRefCacheAware->RunRefSize = RunRefSize;
 630     RunRefCacheAware->Number = Count;
 631
 632     /* There is no allocated pool! */
 633     RunRefCacheAware->PoolToFree = (PVOID)0xBADCA11u;
 634
 635     /* Initialize runref */
 636     if (RunRefCacheAware->Number != 0)
 637     {
 638         for (Count = 0; Count < RunRefCacheAware->Number; ++Count)
 639         {
 640             Offset = RunRefCacheAware->RunRefSize * Count;
 641             RunRef = (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs + Offset);
 642             RunRef->Count = 0;
 643         }
 644     }
 645 }
 646
 647 /*
 648  * @implemented NT5.2
 649  */
 650 SIZE_T
 651 NTAPI
 652 ExSizeOfRundownProtectionCacheAware(VOID)
 653 {
 654     SIZE_T Size;
 655
 656     PAGED_CODE();
 657
 658     /* Compute the needed size for runrefs */
 659     if (KeNumberProcessors <= 1)
 660     {
 661         Size = sizeof(EX_RUNDOWN_REF);
 662     }
 663     else
 664     {
 665         /* We +1, to have enough room for alignment */
 666         Size = (KeNumberProcessors + 1) * KeGetRecommendedSharedDataAlignment();
 667     }
 668
 669     /* Return total size (master structure and runrefs) */
 670     return Size + sizeof(EX_RUNDOWN_REF_CACHE_AWARE);
 671 }
 672