reactos/ntoskrnl/include/internal/ex.h

   1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
   2 #define __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
   3
   4 /* GLOBAL VARIABLES *********************************************************/
   5
   6 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   7 extern LARGE_INTEGER ExpTimeZoneBias;
   8 extern ULONG ExpTimeZoneId;
   9 extern ULONG ExpTickCountMultiplier;
  10 extern ULONG ExpLastTimeZoneBias;
  11 extern POBJECT_TYPE ExEventPairObjectType;
  12 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  13 extern ULONG NtBuildNumber;
  14 extern ULONG NtMajorVersion;
  15 extern ULONG NtMinorVersion;
  16 extern FAST_MUTEX ExpEnvironmentLock;
  17 extern ERESOURCE ExpFirmwareTableResource;
  18 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  19 extern BOOLEAN ExpIsWinPEMode;
  20 extern LIST_ENTRY ExpSystemResourcesList;
  21 ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  22 ULONG ExpUnicodeCaseTableDataOffset;
  23 PVOID ExpNlsSectionPointer;
  24 extern CHAR NtBuildLab[];
  25 extern ULONG CmNtCSDVersion;
  26 extern ULONG NtGlobalFlag;
  27 extern ULONG ExpInitializationPhase;
  28 extern ULONG ExpAltTimeZoneBias;
  29 extern LIST_ENTRY ExSystemLookasideListHead;
  30
  31 typedef struct _EXHANDLE
  32 {
  33     union
  34     {
  35         struct
  36         {
  37             ULONG TagBits:2;
  38             ULONG Index:30;
  39         };
  40         HANDLE GenericHandleOverlay;
  41         ULONG_PTR Value;
  42     };
  43 } EXHANDLE, *PEXHANDLE;
  44
  45 typedef struct _ETIMER
  46 {
  47     KTIMER KeTimer;
  48     KAPC TimerApc;
  49     KDPC TimerDpc;
  50     LIST_ENTRY ActiveTimerListEntry;
  51     KSPIN_LOCK Lock;
  52     LONG Period;
  53     BOOLEAN ApcAssociated;
  54     BOOLEAN WakeTimer;
  55     LIST_ENTRY WakeTimerListEntry;
  56 } ETIMER, *PETIMER;
  57
  58 typedef struct
  59 {
  60     PCALLBACK_OBJECT *CallbackObject;
  61     PWSTR Name;
  62 } SYSTEM_CALLBACKS;
  63
  64 #define MAX_FAST_REFS           7
  65
  66 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
  67 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
  68 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
  69 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
  70 #define ExRundownCompleted                              _ExRundownCompleted
  71 #define ExGetPreviousMode                               KeGetPreviousMode
  72
  73
  74 //
  75 // Various bits tagged on the handle or handle table
  76 //
  77 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
  78 #define FREE_HANDLE_MASK                -1
  79
  80 //
  81 // Number of entries in each table level
  82 //
  83 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
  84 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
  85 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
  86
  87 //
  88 // Maximum index in each table level before we need another table
  89 //
  90 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
  91 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
  92 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
  93
  94 //
  95 // Detect old GCC
  96 //
  97 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
  98     (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
  99
 100 #define DEFINE_WAIT_BLOCK(x)                                \
 101     struct _AlignHack                                       \
 102     {                                                       \
 103         UCHAR Hack[15];                                     \
 104         EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock;             \
 105     } WaitBlockBuffer;                                      \
 106     PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
 107         ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
 108
 109 #else
 110
 111 //
 112 // This is only for compatibility; the compiler will optimize the extra
 113 // local variable (the actual pointer) away, so we don't take any perf hit
 114 // by doing this.
 115 //
 116 #define DEFINE_WAIT_BLOCK(x)                                \
 117     EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer;                \
 118     PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
 119
 120 #endif
 121
 122 #ifdef _WIN64
 123 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
 124 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 125 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
 126 #else
 127 #define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
 128 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
 129 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
 130 #endif
 131
 132 /* INITIALIZATION FUNCTIONS *************************************************/
 133
 134 VOID
 135 NTAPI
 136 ExpWin32kInit(VOID);
 137
 138 VOID
 139 NTAPI
 140 ExInit2(VOID);
 141
 142 VOID
 143 NTAPI
 144 Phase1Initialization(
 145     IN PVOID Context
 146 );
 147
 148 VOID
 149 NTAPI
 150 ExpInitializePushLocks(VOID);
 151
 152 BOOLEAN
 153 NTAPI
 154 ExRefreshTimeZoneInformation(
 155     IN PLARGE_INTEGER SystemBootTime
 156 );
 157
 158 VOID
 159 NTAPI
 160 ExpInitializeWorkerThreads(VOID);
 161
 162 VOID
 163 NTAPI
 164 ExpInitLookasideLists(VOID);
 165
 166 VOID
 167 NTAPI
 168 ExInitializeSystemLookasideList(
 169     IN PGENERAL_LOOKASIDE List,
 170     IN POOL_TYPE Type,
 171     IN ULONG Size,
 172     IN ULONG Tag,
 173     IN USHORT MaximumDepth,
 174     IN PLIST_ENTRY ListHead
 175 );
 176
 177 BOOLEAN
 178 NTAPI
 179 ExpInitializeCallbacks(VOID);
 180
 181 VOID
 182 NTAPI
 183 ExpInitUuids(VOID);
 184
 185 VOID
 186 NTAPI
 187 ExpInitializeExecutive(
 188     IN ULONG Cpu,
 189     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 190 );
 191
 192 VOID
 193 NTAPI
 194 ExpInitializeEventImplementation(VOID);
 195
 196 VOID
 197 NTAPI
 198 ExpInitializeEventImplementation(VOID);
 199
 200 VOID
 201 NTAPI
 202 ExpInitializeEventPairImplementation(VOID);
 203
 204 VOID
 205 NTAPI
 206 ExpInitializeSemaphoreImplementation(VOID);
 207
 208 VOID
 209 NTAPI
 210 ExpInitializeMutantImplementation(VOID);
 211
 212 VOID
 213 NTAPI
 214 ExpInitializeTimerImplementation(VOID);
 215
 216 VOID
 217 NTAPI
 218 ExpInitializeProfileImplementation(VOID);
 219
 220 VOID
 221 NTAPI
 222 ExpResourceInitialization(VOID);
 223
 224 VOID
 225 NTAPI
 226 ExInitPoolLookasidePointers(VOID);
 227
 228 /* Callback Functions ********************************************************/
 229
 230 VOID
 231 NTAPI
 232 ExInitializeCallBack(
 233     IN OUT PEX_CALLBACK Callback
 234 );
 235
 236 PEX_CALLBACK_ROUTINE_BLOCK
 237 NTAPI
 238 ExAllocateCallBack(
 239     IN PEX_CALLBACK_FUNCTION Function,
 240     IN PVOID Context
 241 );
 242
 243 VOID
 244 NTAPI
 245 ExFreeCallBack(
 246     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 247 );
 248
 249 BOOLEAN
 250 NTAPI
 251 ExCompareExchangeCallBack (
 252     IN OUT PEX_CALLBACK CallBack,
 253     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 254     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 255 );
 256
 257 PEX_CALLBACK_ROUTINE_BLOCK
 258 NTAPI
 259 ExReferenceCallBackBlock(
 260     IN OUT PEX_CALLBACK CallBack
 261 );
 262
 263 VOID
 264 NTAPI
 265 ExDereferenceCallBackBlock(
 266     IN OUT PEX_CALLBACK CallBack,
 267     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 268 );
 269
 270 PEX_CALLBACK_FUNCTION
 271 NTAPI
 272 ExGetCallBackBlockRoutine(
 273     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 274 );
 275
 276 PVOID
 277 NTAPI
 278 ExGetCallBackBlockContext(
 279     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 280 );
 281
 282 VOID
 283 NTAPI
 284 ExWaitForCallBacks(
 285     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 286 );
 287
 288 /* Rundown Functions ********************************************************/
 289
 290 VOID
 291 FASTCALL
 292 ExfInitializeRundownProtection(
 293      OUT PEX_RUNDOWN_REF RunRef
 294 );
 295
 296 VOID
 297 FASTCALL
 298 ExfReInitializeRundownProtection(
 299      OUT PEX_RUNDOWN_REF RunRef
 300 );
 301
 302 BOOLEAN
 303 FASTCALL
 304 ExfAcquireRundownProtection(
 305      IN OUT PEX_RUNDOWN_REF RunRef
 306 );
 307
 308 BOOLEAN
 309 FASTCALL
 310 ExfAcquireRundownProtectionEx(
 311      IN OUT PEX_RUNDOWN_REF RunRef,
 312      IN ULONG Count
 313 );
 314
 315 VOID
 316 FASTCALL
 317 ExfReleaseRundownProtection(
 318      IN OUT PEX_RUNDOWN_REF RunRef
 319 );
 320
 321 VOID
 322 FASTCALL
 323 ExfReleaseRundownProtectionEx(
 324      IN OUT PEX_RUNDOWN_REF RunRef,
 325      IN ULONG Count
 326 );
 327
 328 VOID
 329 FASTCALL
 330 ExfRundownCompleted(
 331      OUT PEX_RUNDOWN_REF RunRef
 332 );
 333
 334 VOID
 335 FASTCALL
 336 ExfWaitForRundownProtectionRelease(
 337      IN OUT PEX_RUNDOWN_REF RunRef
 338 );
 339
 340 /* HANDLE TABLE FUNCTIONS ***************************************************/
 341
 342 typedef BOOLEAN
 343 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 344     PHANDLE_TABLE_ENTRY HandleTableEntry,
 345     HANDLE Handle,
 346     PVOID Context
 347 );
 348
 349 typedef BOOLEAN
 350 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 351     IN PEPROCESS Process,
 352     IN PHANDLE_TABLE HandleTable,
 353     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 354     IN PHANDLE_TABLE_ENTRY NewEntry
 355 );
 356
 357 typedef BOOLEAN
 358 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 359     PHANDLE_TABLE_ENTRY HandleTableEntry,
 360     ULONG_PTR Context
 361 );
 362
 363 VOID
 364 NTAPI
 365 ExpInitializeHandleTables(
 366     VOID
 367 );
 368
 369 PHANDLE_TABLE
 370 NTAPI
 371 ExCreateHandleTable(
 372     IN PEPROCESS Process OPTIONAL
 373 );
 374
 375 VOID
 376 NTAPI
 377 ExUnlockHandleTableEntry(
 378     IN PHANDLE_TABLE HandleTable,
 379     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 380 );
 381
 382 HANDLE
 383 NTAPI
 384 ExCreateHandle(
 385     IN PHANDLE_TABLE HandleTable,
 386     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 387 );
 388
 389 VOID
 390 NTAPI
 391 ExDestroyHandleTable(
 392     IN PHANDLE_TABLE HandleTable,
 393     IN PVOID DestroyHandleProcedure OPTIONAL
 394 );
 395
 396 BOOLEAN
 397 NTAPI
 398 ExDestroyHandle(
 399     IN PHANDLE_TABLE HandleTable,
 400     IN HANDLE Handle,
 401     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 402 );
 403
 404 PHANDLE_TABLE_ENTRY
 405 NTAPI
 406 ExMapHandleToPointer(
 407     IN PHANDLE_TABLE HandleTable,
 408     IN HANDLE Handle
 409 );
 410
 411 PHANDLE_TABLE
 412 NTAPI
 413 ExDupHandleTable(
 414     IN PEPROCESS Process,
 415     IN PHANDLE_TABLE HandleTable,
 416     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 417     IN ULONG_PTR Mask
 418 );
 419
 420 BOOLEAN
 421 NTAPI
 422 ExChangeHandle(
 423     IN PHANDLE_TABLE HandleTable,
 424     IN HANDLE Handle,
 425     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 426     IN ULONG_PTR Context
 427 );
 428
 429 VOID
 430 NTAPI
 431 ExSweepHandleTable(
 432     IN PHANDLE_TABLE HandleTable,
 433     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 434     IN PVOID Context
 435 );
 436
 437 /* PSEH EXCEPTION HANDLING **************************************************/
 438
 439 LONG
 440 NTAPI
 441 ExSystemExceptionFilter(VOID);
 442
 443 /* CALLBACKS *****************************************************************/
 444
 445 FORCEINLINE
 446 VOID
 447 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 448              IN PVOID Context,
 449              IN PVOID Argument1,
 450              IN PVOID Argument2)
 451 {
 452     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 453     PEX_CALLBACK_FUNCTION Function;
 454
 455     /* Reference the block */
 456     CallbackBlock = ExReferenceCallBackBlock(Callback);
 457     if (CallbackBlock)
 458     {
 459         /* Get the function */
 460         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 461
 462         /* Do the callback */
 463         Function(Context, Argument1, Argument2);
 464
 465         /* Now dereference it */
 466         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 467     }
 468 }
 469
 470 /* FAST REFS ******************************************************************/
 471
 472 FORCEINLINE
 473 PVOID
 474 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 475 {
 476     /* Return the unbiased pointer */
 477     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 478 }
 479
 480 FORCEINLINE
 481 ULONG
 482 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 483 {
 484     /* Return the reference count */
 485     return FastRef.RefCnt;
 486 }
 487
 488 FORCEINLINE
 489 VOID
 490 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 491                           IN OPTIONAL PVOID Object)
 492 {
 493     /* Sanity check */
 494     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 495
 496     /* Check if an object is being set */
 497     if (!Object)
 498     {
 499         /* Clear the field */
 500         FastRef->Object = NULL;
 501     }
 502     else
 503     {
 504         /* Otherwise, we assume the object was referenced and is ready */
 505         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 506     }
 507 }
 508
 509 FORCEINLINE
 510 EX_FAST_REF
 511 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 512 {
 513     EX_FAST_REF OldValue, NewValue;
 514
 515     /* Start reference loop */
 516     for (;;)
 517     {
 518         /* Get the current reference count */
 519         OldValue = *FastRef;
 520         if (OldValue.RefCnt)
 521         {
 522             /* Increase the reference count */
 523             NewValue.Value = OldValue.Value - 1;
 524             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 525                                                 NewValue.Object,
 526                                                 OldValue.Object);
 527             if (NewValue.Object != OldValue.Object) continue;
 528         }
 529
 530         /* We are done */
 531         break;
 532     }
 533
 534     /* Return the old value */
 535     return OldValue;
 536 }
 537
 538 FORCEINLINE
 539 BOOLEAN
 540 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 541                       IN PVOID Object)
 542 {
 543     EX_FAST_REF OldValue, NewValue;
 544
 545     /* Sanity checks */
 546     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 547
 548     /* Start update loop */
 549     for (;;)
 550     {
 551         /* Get the current reference count */
 552         OldValue = *FastRef;
 553
 554         /* Check if the current count is too high or if the pointer changed */
 555         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 556             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 557         {
 558             /* Fail */
 559             return FALSE;
 560         }
 561
 562         /* Update the reference count */
 563         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 564         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 565                                             NewValue.Object,
 566                                             OldValue.Object);
 567         if (NewValue.Object != OldValue.Object) continue;
 568
 569         /* We are done */
 570         break;
 571     }
 572
 573     /* Return success */
 574     return TRUE;
 575 }
 576
 577 FORCEINLINE
 578 BOOLEAN
 579 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 580                        IN PVOID Object)
 581 {
 582     EX_FAST_REF OldValue, NewValue;
 583
 584     /* Sanity checks */
 585     ASSERT(Object != NULL);
 586     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 587
 588     /* Start reference loop */
 589     for (;;)
 590     {
 591         /* Get the current reference count */
 592         OldValue = *FastRef;
 593
 594         /* Check if we're full if if the pointer changed */
 595         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 596
 597         /* Decrease the reference count */
 598         NewValue.Value = OldValue.Value + 1;
 599         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 600                                             NewValue.Object,
 601                                             OldValue.Object);
 602         if (NewValue.Object != OldValue.Object) continue;
 603
 604         /* We are done */
 605         break;
 606     }
 607
 608     /* Return success */
 609     return TRUE;
 610 }
 611
 612 FORCEINLINE
 613 EX_FAST_REF
 614 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 615                     IN PVOID Object)
 616 {
 617     EX_FAST_REF NewValue, OldValue;
 618
 619     /* Sanity check */
 620     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 621
 622     /* Check if an object is being set */
 623     if (!Object)
 624     {
 625         /* Clear the field */
 626         NewValue.Object = NULL;
 627     }
 628     else
 629     {
 630         /* Otherwise, we assume the object was referenced and is ready */
 631         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 632     }
 633
 634     /* Update the object */
 635     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 636     return OldValue;
 637 }
 638
 639 FORCEINLINE
 640 EX_FAST_REF
 641 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 642                            IN PVOID Object,
 643                            IN PVOID OldObject)
 644 {
 645     EX_FAST_REF OldValue, NewValue;
 646
 647     /* Sanity check and start swap loop */
 648     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 649     for (;;)
 650     {
 651         /* Get the current value */
 652         OldValue = *FastRef;
 653
 654         /* Make sure there's enough references to swap */
 655         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 656
 657         /* Check if we have an object to swap */
 658         if (Object)
 659         {
 660             /* Set up the value with maximum fast references */
 661             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 662         }
 663         else
 664         {
 665             /* Write the object address itself (which is empty) */
 666             NewValue.Value = (ULONG_PTR)Object;
 667         }
 668
 669         /* Do the actual compare exchange */
 670         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 671                                             NewValue.Object,
 672                                             OldValue.Object);
 673         if (NewValue.Object != OldValue.Object) continue;
 674
 675         /* All done */
 676         break;
 677     }
 678
 679     /* Return the old value */
 680     return OldValue;
 681 }
 682
 683 /* RUNDOWN *******************************************************************/
 684
 685 /*++
 686  * @name ExfAcquireRundownProtection
 687  * INTERNAL MACRO
 688  *
 689  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 690  *     the specified descriptor.
 691  *
 692  * @param RunRef
 693  *        Pointer to a rundown reference descriptor.
 694  *
 695  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 696  *
 697  * @remarks This is the internal macro for system use only.In case the rundown
 698  *          was active, then the slow-path will be called through the exported
 699  *          function.
 700  *
 701  *--*/
 702 FORCEINLINE
 703 BOOLEAN
 704 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 705 {
 706     ULONG_PTR Value, NewValue;
 707
 708     /* Get the current value and mask the active bit */
 709     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 710
 711     /* Add a reference */
 712     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 713
 714     /* Change the value */
 715     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 716     if (NewValue != Value)
 717     {
 718         /* Rundown was active, use long path */
 719         return ExfAcquireRundownProtection(RunRef);
 720     }
 721
 722     /* Success */
 723     return TRUE;
 724 }
 725
 726 /*++
 727  * @name ExReleaseRundownProtection
 728  * INTERNAL MACRO
 729  *
 730  *     The ExReleaseRundownProtection routine releases rundown protection for
 731  *     the specified descriptor.
 732  *
 733  * @param RunRef
 734  *        Pointer to a rundown reference descriptor.
 735  *
 736  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 737  *
 738  * @remarks This is the internal macro for system use only.In case the rundown
 739  *          was active, then the slow-path will be called through the exported
 740  *          function.
 741  *
 742  *--*/
 743 FORCEINLINE
 744 VOID
 745 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 746 {
 747     ULONG_PTR Value, NewValue;
 748
 749     /* Get the current value and mask the active bit */
 750     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 751
 752     /* Remove a reference */
 753     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 754
 755     /* Change the value */
 756     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 757
 758     /* Check if the rundown was active */
 759     if (NewValue != Value)
 760     {
 761         /* Rundown was active, use long path */
 762         ExfReleaseRundownProtection(RunRef);
 763     }
 764     else
 765     {
 766         /* Sanity check */
 767         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 768     }
 769 }
 770
 771 /*++
 772  * @name ExInitializeRundownProtection
 773  * INTERNAL MACRO
 774  *
 775  *     The ExInitializeRundownProtection routine initializes a rundown
 776  *     protection descriptor.
 777  *
 778  * @param RunRef
 779  *        Pointer to a rundown reference descriptor.
 780  *
 781  * @return None.
 782  *
 783  * @remarks This is the internal macro for system use only.
 784  *
 785  *--*/
 786 FORCEINLINE
 787 VOID
 788 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 789 {
 790     /* Set the count to zero */
 791     RunRef->Count = 0;
 792 }
 793
 794 /*++
 795  * @name ExWaitForRundownProtectionRelease
 796  * INTERNAL MACRO
 797  *
 798  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 799  *     rundown descriptor has been released.
 800  *
 801  * @param RunRef
 802  *        Pointer to a rundown reference descriptor.
 803  *
 804  * @return None.
 805  *
 806  * @remarks This is the internal macro for system use only. If a wait is actually
 807  *          necessary, then the slow path is taken through the exported function.
 808  *
 809  *--*/
 810 FORCEINLINE
 811 VOID
 812 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 813 {
 814     ULONG_PTR Value;
 815
 816     /* Set the active bit */
 817     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 818     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 819     {
 820         /* If the the rundown wasn't already active, then take the long path */
 821         ExfWaitForRundownProtectionRelease(RunRef);
 822     }
 823 }
 824
 825 /*++
 826  * @name ExRundownCompleted
 827  * INTERNAL MACRO
 828  *
 829  *     The ExRundownCompleted routine completes the rundown of the specified
 830  *     descriptor by setting the active bit.
 831  *
 832  * @param RunRef
 833  *        Pointer to a rundown reference descriptor.
 834  *
 835  * @return None.
 836  *
 837  * @remarks This is the internal macro for system use only.
 838  *
 839  *--*/
 840 FORCEINLINE
 841 VOID
 842 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 843 {
 844     /* Sanity check */
 845     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 846
 847     /* Mark the counter as active */
 848     ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
 849 }
 850
 851 /* PUSHLOCKS *****************************************************************/
 852
 853 /* FIXME: VERIFY THESE! */
 854
 855 VOID
 856 FASTCALL
 857 ExBlockPushLock(
 858     IN PEX_PUSH_LOCK PushLock,
 859     IN PVOID WaitBlock
 860 );
 861
 862 VOID
 863 FASTCALL
 864 ExfUnblockPushLock(
 865     IN PEX_PUSH_LOCK PushLock,
 866     IN PVOID CurrentWaitBlock
 867 );
 868
 869 VOID
 870 FASTCALL
 871 ExWaitForUnblockPushLock(
 872     IN PEX_PUSH_LOCK PushLock,
 873     IN PVOID WaitBlock
 874 );
 875
 876 /*++
 877  * @name ExInitializePushLock
 878  * INTERNAL MACRO
 879  *
 880  *     The ExInitializePushLock macro initializes a PushLock.
 881  *
 882  * @params PushLock
 883  *         Pointer to the pushlock which is to be initialized.
 884  *
 885  * @return None.
 886  *
 887  * @remarks None.
 888  *
 889  *--*/
 890 FORCEINLINE
 891 VOID
 892 ExInitializePushLock(IN PULONG_PTR PushLock)
 893 {
 894     /* Set the value to 0 */
 895     *PushLock = 0;
 896 }
 897
 898 /*++
 899  * @name ExAcquirePushLockExclusive
 900  * INTERNAL MACRO
 901  *
 902  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 903  *
 904  * @params PushLock
 905  *         Pointer to the pushlock which is to be acquired.
 906  *
 907  * @return None.
 908  *
 909  * @remarks The function attempts the quickest route to acquire the lock, which is
 910  *          to simply set the lock bit.
 911  *          However, if the pushlock is already shared, the slower path is taken.
 912  *
 913  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 914  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 915  *
 916  *--*/
 917 FORCEINLINE
 918 VOID
 919 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 920 {
 921     /* Try acquiring the lock */
 922     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 923     {
 924         /* Someone changed it, use the slow path */
 925         ExfAcquirePushLockExclusive(PushLock);
 926     }
 927
 928     /* Sanity check */
 929     ASSERT(PushLock->Locked);
 930 }
 931
 932 /*++
 933 * @name ExTryToAcquirePushLockExclusive
 934 * INTERNAL MACRO
 935 *
 936 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 937 *
 938 * @params PushLock
 939 *         Pointer to the pushlock which is to be acquired.
 940 *
 941 * @return None.
 942 *
 943 * @remarks The function attempts the quickest route to acquire the lock, which is
 944 *          to simply set the lock bit.
 945 *          However, if the pushlock is already shared, the slower path is taken.
 946 *
 947 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 948 *          This macro should usually be paired up with KeAcquireCriticalRegion.
 949 *
 950 *--*/
 951 FORCEINLINE
 952 BOOLEAN
 953 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 954 {
 955     /* Try acquiring the lock */
 956     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 957     {
 958         /* Can't acquire */
 959         return FALSE;
 960     }
 961
 962     /* Got acquired */
 963     ASSERT (PushLock->Locked);
 964     return TRUE;
 965 }
 966
 967 /*++
 968  * @name ExAcquirePushLockShared
 969  * INTERNAL MACRO
 970  *
 971  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
 972  *
 973  * @params PushLock
 974  *         Pointer to the pushlock which is to be acquired.
 975  *
 976  * @return None.
 977  *
 978  * @remarks The function attempts the quickest route to acquire the lock, which is
 979  *          to simply set the lock bit and set the share count to one.
 980  *          However, if the pushlock is already shared, the slower path is taken.
 981  *
 982  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 983  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 984  *
 985  *--*/
 986 FORCEINLINE
 987 VOID
 988 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
 989 {
 990     EX_PUSH_LOCK NewValue;
 991
 992     /* Try acquiring the lock */
 993     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
 994     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
 995     {
 996         /* Someone changed it, use the slow path */
 997         ExfAcquirePushLockShared(PushLock);
 998     }
 999
1000     /* Sanity checks */
1001     ASSERT(PushLock->Locked);
1002     ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1003 }
1004
1005 /*++
1006  * @name ExConvertPushLockSharedToExclusive
1007  * INTERNAL MACRO
1008  *
1009  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1010  *     pushlock to a shared pushlock.
1011  *
1012  * @params PushLock
1013  *         Pointer to the pushlock which is to be converted.
1014  *
1015  * @return FALSE if conversion failed, TRUE otherwise.
1016  *
1017  * @remarks The function attempts the quickest route to convert the lock, which is
1018  *          to simply set the lock bit and remove any other bits.
1019  *
1020  *--*/
1021 FORCEINLINE
1022 BOOLEAN
1023 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1024 {
1025     EX_PUSH_LOCK OldValue;
1026
1027     /* Set the expected old value */
1028     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1029
1030     /* Try converting the lock */
1031     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1032         OldValue.Ptr)
1033     {
1034         /* Conversion failed */
1035         return FALSE;
1036     }
1037
1038     /* Sanity check */
1039     ASSERT(PushLock->Locked);
1040     return TRUE;
1041 }
1042
1043 /*++
1044  * @name ExWaitOnPushLock
1045  * INTERNAL MACRO
1046  *
1047  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1048  *
1049  * @params PushLock
1050  *         Pointer to a pushlock.
1051  *
1052  * @return None.
1053  *
1054  * @remarks The function attempts to get any exclusive waiters out of their slow
1055  *          path by forcing an instant acquire/release operation.
1056  *
1057  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1058  *
1059  *--*/
1060 FORCEINLINE
1061 VOID
1062 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1063 {
1064     /* Check if we're locked */
1065     if (PushLock->Locked)
1066     {
1067         /* Acquire the lock */
1068         ExfAcquirePushLockExclusive(PushLock);
1069         ASSERT(PushLock->Locked);
1070
1071         /* Release it */
1072         ExfReleasePushLockExclusive(PushLock);
1073     }
1074 }
1075
1076 /*++
1077  * @name ExReleasePushLockShared
1078  * INTERNAL MACRO
1079  *
1080  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1081  *
1082  * @params PushLock
1083  *         Pointer to a previously acquired pushlock.
1084  *
1085  * @return None.
1086  *
1087  * @remarks The function attempts the quickest route to release the lock, which is
1088  *          to simply decrease the share count and remove the lock bit.
1089  *          However, if the pushlock is being waited on then the long path is taken.
1090  *
1091  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1092  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1093  *
1094  *--*/
1095 FORCEINLINE
1096 VOID
1097 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1098 {
1099     EX_PUSH_LOCK OldValue;
1100
1101     /* Sanity checks */
1102     ASSERT(PushLock->Locked);
1103     ASSERT(PushLock->Waiting || PushLock->Shared > 0);
1104
1105     /* Try to clear the pushlock */
1106     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1107     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1108     {
1109         /* There are still other people waiting on it */
1110         ExfReleasePushLockShared(PushLock);
1111     }
1112 }
1113
1114 /*++
1115  * @name ExReleasePushLockExclusive
1116  * INTERNAL MACRO
1117  *
1118  *     The ExReleasePushLockExclusive macro releases a previously
1119  *     exclusively acquired PushLock.
1120  *
1121  * @params PushLock
1122  *         Pointer to a previously acquired pushlock.
1123  *
1124  * @return None.
1125  *
1126  * @remarks The function attempts the quickest route to release the lock, which is
1127  *          to simply clear the locked bit.
1128  *          However, if the pushlock is being waited on, the slow path is taken
1129  *          in an attempt to wake up the lock.
1130  *
1131  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1132  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1133  *
1134  *--*/
1135 FORCEINLINE
1136 VOID
1137 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1138 {
1139     EX_PUSH_LOCK OldValue;
1140
1141     /* Sanity checks */
1142     ASSERT(PushLock->Locked);
1143     ASSERT(PushLock->Waiting || PushLock->Shared == 0);
1144
1145     /* Unlock the pushlock */
1146     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1147                                                  -(SIZE_T)EX_PUSH_LOCK_LOCK);
1148
1149     /* Sanity checks */
1150     ASSERT(OldValue.Locked);
1151     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1152
1153     /* Check if anyone is waiting on it and it's not already waking*/
1154     if ((OldValue.Waiting) && !(OldValue.Waking))
1155     {
1156         /* Wake it up */
1157         ExfTryToWakePushLock(PushLock);
1158     }
1159 }
1160
1161 /*++
1162  * @name ExReleasePushLock
1163  * INTERNAL MACRO
1164  *
1165  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1166  *
1167  * @params PushLock
1168  *         Pointer to a previously acquired pushlock.
1169  *
1170  * @return None.
1171  *
1172  * @remarks The function attempts the quickest route to release the lock, which is
1173  *          to simply clear all the fields and decrease the share count if required.
1174  *          However, if the pushlock is being waited on then the long path is taken.
1175  *
1176  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1177  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1178  *
1179  *--*/
1180 FORCEINLINE
1181 VOID
1182 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1183 {
1184     EX_PUSH_LOCK OldValue = *PushLock;
1185     EX_PUSH_LOCK NewValue;
1186
1187     /* Sanity checks */
1188     ASSERT(OldValue.Locked);
1189
1190     /* Check if the pushlock is shared */
1191     if (OldValue.Shared > 1)
1192     {
1193         /* Decrease the share count */
1194         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1195     }
1196     else
1197     {
1198         /* Clear the pushlock entirely */
1199         NewValue.Value = 0;
1200     }
1201
1202     /* Check if nobody is waiting on us and try clearing the lock here */
1203     if ((OldValue.Waiting) ||
1204         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1205          OldValue.Ptr))
1206     {
1207         /* We have waiters, use the long path */
1208         ExfReleasePushLock(PushLock);
1209     }
1210 }
1211
1212 /* FAST MUTEX INLINES *********************************************************/
1213
1214 FORCEINLINE
1215 VOID
1216 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1217 {
1218     PKTHREAD Thread = KeGetCurrentThread();
1219
1220     /* Sanity check */
1221     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1222            (Thread->CombinedApcDisable != 0) ||
1223            (Thread->Teb == NULL) ||
1224            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1225     ASSERT(FastMutex->Owner != Thread);
1226
1227     /* Decrease the count */
1228     if (InterlockedDecrement(&FastMutex->Count))
1229     {
1230         /* Someone is still holding it, use slow path */
1231         KiAcquireFastMutex(FastMutex);
1232     }
1233
1234     /* Set the owner */
1235     FastMutex->Owner = Thread;
1236 }
1237
1238 FORCEINLINE
1239 VOID
1240 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1241 {
1242     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1243            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1244            (KeGetCurrentThread()->Teb == NULL) ||
1245            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1246     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1247
1248     /* Erase the owner */
1249     FastMutex->Owner = NULL;
1250
1251     /* Increase the count */
1252     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1253     {
1254         /* Someone was waiting for it, signal the waiter */
1255         KeSetEventBoostPriority(&FastMutex->Gate, NULL);
1256     }
1257 }
1258
1259 FORCEINLINE
1260 VOID
1261 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1262 {
1263     KIRQL OldIrql;
1264     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1265
1266     /* Raise IRQL to APC */
1267     KeRaiseIrql(APC_LEVEL, &OldIrql);
1268
1269     /* Decrease the count */
1270     if (InterlockedDecrement(&FastMutex->Count))
1271     {
1272         /* Someone is still holding it, use slow path */
1273         KiAcquireFastMutex(FastMutex);
1274     }
1275
1276     /* Set the owner and IRQL */
1277     FastMutex->Owner = KeGetCurrentThread();
1278     FastMutex->OldIrql = OldIrql;
1279 }
1280
1281 FORCEINLINE
1282 VOID
1283 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1284 {
1285     KIRQL OldIrql;
1286     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1287
1288     /* Erase the owner */
1289     FastMutex->Owner = NULL;
1290     OldIrql = (KIRQL)FastMutex->OldIrql;
1291
1292     /* Increase the count */
1293     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1294     {
1295         /* Someone was waiting for it, signal the waiter */
1296         KeSetEventBoostPriority(&FastMutex->Gate, NULL);
1297     }
1298
1299     /* Lower IRQL back */
1300     KeLowerIrql(OldIrql);
1301 }
1302
1303 FORCEINLINE
1304 BOOLEAN
1305 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1306 {
1307     KIRQL OldIrql;
1308     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1309
1310     /* Raise to APC_LEVEL */
1311     KeRaiseIrql(APC_LEVEL, &OldIrql);
1312
1313     /* Check if we can quickly acquire it */
1314     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1315     {
1316         /* We have, set us as owners */
1317         FastMutex->Owner = KeGetCurrentThread();
1318         FastMutex->OldIrql = OldIrql;
1319         return TRUE;
1320     }
1321     else
1322     {
1323         /* Acquire attempt failed */
1324         KeLowerIrql(OldIrql);
1325         YieldProcessor();
1326         return FALSE;
1327     }
1328 }
1329
1330 FORCEINLINE
1331 VOID
1332 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1333 {
1334     /* Enter the Critical Region */
1335     KeEnterCriticalRegion();
1336
1337     /* Acquire the mutex unsafely */
1338     _ExAcquireFastMutexUnsafe(FastMutex);
1339 }
1340
1341 FORCEINLINE
1342 VOID
1343 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1344 {
1345     /* Release the mutex unsafely */
1346     _ExReleaseFastMutexUnsafe(FastMutex);
1347
1348     /* Leave the critical region */
1349     KeLeaveCriticalRegion();
1350 }
1351
1352 /* OTHER FUNCTIONS **********************************************************/
1353
1354 BOOLEAN
1355 NTAPI
1356 ExTryToAcquireResourceExclusiveLite(
1357     IN PERESOURCE Resource
1358 );
1359
1360 NTSTATUS
1361 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation);
1362
1363 BOOLEAN
1364 NTAPI
1365 ExAcquireTimeRefreshLock(BOOLEAN Wait);
1366
1367 VOID
1368 NTAPI
1369 ExReleaseTimeRefreshLock(VOID);
1370
1371 VOID
1372 NTAPI
1373 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime,
1374                            IN ULONG MaxSepInSeconds);
1375
1376 NTSTATUS
1377 NTAPI
1378 ExpAllocateLocallyUniqueId(OUT LUID *LocallyUniqueId);
1379
1380 VOID
1381 NTAPI
1382 ExTimerRundown(VOID);
1383
1384 VOID
1385 NTAPI
1386 HeadlessInit(
1387     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1388 );
1389
1390 VOID
1391 NTAPI
1392 XIPInit(
1393     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1394 );
1395
1396 #define InterlockedDecrementUL(Addend) \
1397    (ULONG)InterlockedDecrement((PLONG)(Addend))
1398
1399 #define InterlockedIncrementUL(Addend) \
1400    (ULONG)InterlockedIncrement((PLONG)(Addend))
1401
1402 #define InterlockedExchangeUL(Target, Value) \
1403    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1404
1405 #define InterlockedExchangeAddUL(Addend, Value) \
1406    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1407
1408 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1409    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1410
1411 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1412    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))
1413
1414 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H */