ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31
  32 /*
  33  * NT/Cm Version Info variables
  34  */
  35 extern ULONG NtMajorVersion;
  36 extern ULONG NtMinorVersion;
  37 extern ULONG NtBuildNumber;
  38 extern ULONG CmNtSpBuildNumber;
  39 extern ULONG CmNtCSDVersion;
  40 extern ULONG CmNtCSDReleaseType;
  41 extern UNICODE_STRING CmVersionString;
  42 extern UNICODE_STRING CmCSDVersionString;
  43 extern CHAR NtBuildLab[];
  44
  45 // #ifdef _WINKD_
  46 /*
  47  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  48  */
  49 typedef enum _WINKD_WORKER_STATE
  50 {
  51     WinKdWorkerReady = 0,
  52     WinKdWorkerStart,
  53     WinKdWorkerInitialized
  54 } WINKD_WORKER_STATE;
  55
  56 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  57 extern WINKD_WORKER_STATE ExpDebuggerWork;
  58 extern PEPROCESS ExpDebuggerProcessAttach;
  59 extern PEPROCESS ExpDebuggerProcessKill;
  60 extern ULONG_PTR ExpDebuggerPageIn;
  61 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  62 // #endif /* _WINKD_ */
  63
  64 #ifdef _WIN64
  65 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  66 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  67 #else
  68 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  69 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  70 #endif
  71 #define HANDLE_TAG_BITS (2)
  72 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
  73 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
  74
  75 typedef union _EXHANDLE
  76 {
  77      struct
  78      {
  79          ULONG_PTR TagBits:     HANDLE_TAG_BITS;
  80          ULONG_PTR Index:       HANDLE_INDEX_BITS;
  81          ULONG_PTR KernelFlag : KERNEL_FLAG_BITS;
  82      };
  83      struct
  84      {
  85          ULONG_PTR TagBits2:    HANDLE_TAG_BITS;
  86          ULONG_PTR LowIndex:    HANDLE_LOW_BITS;
  87          ULONG_PTR MidIndex:    HANDLE_HIGH_BITS;
  88          ULONG_PTR HighIndex:   HANDLE_HIGH_BITS;
  89          ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
  90      };
  91      HANDLE GenericHandleOverlay;
  92      ULONG_PTR Value;
  93      ULONG AsULONG;
  94 } EXHANDLE, *PEXHANDLE;
  95
  96 typedef struct _ETIMER
  97 {
  98     KTIMER KeTimer;
  99     KAPC TimerApc;
 100     KDPC TimerDpc;
 101     LIST_ENTRY ActiveTimerListEntry;
 102     KSPIN_LOCK Lock;
 103     LONG Period;
 104     BOOLEAN ApcAssociated;
 105     BOOLEAN WakeTimer;
 106     LIST_ENTRY WakeTimerListEntry;
 107 } ETIMER, *PETIMER;
 108
 109 typedef struct
 110 {
 111     PCALLBACK_OBJECT *CallbackObject;
 112     PWSTR Name;
 113 } SYSTEM_CALLBACKS;
 114
 115 typedef struct _HARDERROR_USER_PARAMETERS
 116 {
 117     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 118     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 119     WCHAR Buffer[ANYSIZE_ARRAY];
 120 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 121
 122 #define MAX_FAST_REFS           7
 123
 124 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 125 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 126 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 127 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 128 #define ExRundownCompleted                              _ExRundownCompleted
 129 #define ExGetPreviousMode                               KeGetPreviousMode
 130
 131
 132 //
 133 // Various bits tagged on the handle or handle table
 134 //
 135 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 136 #define FREE_HANDLE_MASK                -1
 137
 138 //
 139 // Number of entries in each table level
 140 //
 141 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 142 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 143 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 144
 145 //
 146 // Maximum index in each table level before we need another table
 147 //
 148 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 149 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 150 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 151
 152 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 153 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 154 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 155
 156 NTSTATUS
 157 NTAPI
 158 ExGetPoolTagInfo(
 159     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 160     IN ULONG SystemInformationLength,
 161     IN OUT PULONG ReturnLength OPTIONAL
 162 );
 163
 164 /* INITIALIZATION FUNCTIONS *************************************************/
 165
 166 BOOLEAN
 167 NTAPI
 168 ExpWin32kInit(VOID);
 169
 170 VOID
 171 NTAPI
 172 ExInit2(VOID);
 173
 174 VOID
 175 NTAPI
 176 Phase1Initialization(
 177     IN PVOID Context
 178 );
 179
 180 VOID
 181 NTAPI
 182 ExpInitializePushLocks(VOID);
 183
 184 BOOLEAN
 185 NTAPI
 186 ExRefreshTimeZoneInformation(
 187     IN PLARGE_INTEGER SystemBootTime
 188 );
 189
 190 VOID
 191 NTAPI
 192 ExpInitializeWorkerThreads(VOID);
 193
 194 VOID
 195 NTAPI
 196 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 197
 198 VOID
 199 NTAPI
 200 ExpInitLookasideLists(VOID);
 201
 202 VOID
 203 NTAPI
 204 ExInitializeSystemLookasideList(
 205     IN PGENERAL_LOOKASIDE List,
 206     IN POOL_TYPE Type,
 207     IN ULONG Size,
 208     IN ULONG Tag,
 209     IN USHORT MaximumDepth,
 210     IN PLIST_ENTRY ListHead
 211 );
 212
 213 BOOLEAN
 214 NTAPI
 215 ExpInitializeCallbacks(VOID);
 216
 217 VOID
 218 NTAPI
 219 ExpInitUuids(VOID);
 220
 221 VOID
 222 NTAPI
 223 ExpInitializeExecutive(
 224     IN ULONG Cpu,
 225     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 226 );
 227
 228 BOOLEAN
 229 NTAPI
 230 ExpInitializeEventImplementation(VOID);
 231
 232 BOOLEAN
 233 NTAPI
 234 ExpInitializeKeyedEventImplementation(VOID);
 235
 236 BOOLEAN
 237 NTAPI
 238 ExpInitializeEventPairImplementation(VOID);
 239
 240 BOOLEAN
 241 NTAPI
 242 ExpInitializeSemaphoreImplementation(VOID);
 243
 244 BOOLEAN
 245 NTAPI
 246 ExpInitializeMutantImplementation(VOID);
 247
 248 BOOLEAN
 249 NTAPI
 250 ExpInitializeTimerImplementation(VOID);
 251
 252 BOOLEAN
 253 NTAPI
 254 ExpInitializeProfileImplementation(VOID);
 255
 256 VOID
 257 NTAPI
 258 ExpResourceInitialization(VOID);
 259
 260 VOID
 261 NTAPI
 262 ExInitPoolLookasidePointers(VOID);
 263
 264 /* Callback Functions ********************************************************/
 265
 266 VOID
 267 NTAPI
 268 ExInitializeCallBack(
 269     IN OUT PEX_CALLBACK Callback
 270 );
 271
 272 PEX_CALLBACK_ROUTINE_BLOCK
 273 NTAPI
 274 ExAllocateCallBack(
 275     IN PEX_CALLBACK_FUNCTION Function,
 276     IN PVOID Context
 277 );
 278
 279 VOID
 280 NTAPI
 281 ExFreeCallBack(
 282     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 283 );
 284
 285 BOOLEAN
 286 NTAPI
 287 ExCompareExchangeCallBack (
 288     IN OUT PEX_CALLBACK CallBack,
 289     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 290     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 291 );
 292
 293 PEX_CALLBACK_ROUTINE_BLOCK
 294 NTAPI
 295 ExReferenceCallBackBlock(
 296     IN OUT PEX_CALLBACK CallBack
 297 );
 298
 299 VOID
 300 NTAPI
 301 ExDereferenceCallBackBlock(
 302     IN OUT PEX_CALLBACK CallBack,
 303     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 304 );
 305
 306 PEX_CALLBACK_FUNCTION
 307 NTAPI
 308 ExGetCallBackBlockRoutine(
 309     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 310 );
 311
 312 PVOID
 313 NTAPI
 314 ExGetCallBackBlockContext(
 315     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 316 );
 317
 318 VOID
 319 NTAPI
 320 ExWaitForCallBacks(
 321     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 322 );
 323
 324 /* Rundown Functions ********************************************************/
 325
 326 VOID
 327 FASTCALL
 328 ExfInitializeRundownProtection(
 329      OUT PEX_RUNDOWN_REF RunRef
 330 );
 331
 332 VOID
 333 FASTCALL
 334 ExfReInitializeRundownProtection(
 335      OUT PEX_RUNDOWN_REF RunRef
 336 );
 337
 338 BOOLEAN
 339 FASTCALL
 340 ExfAcquireRundownProtection(
 341      IN OUT PEX_RUNDOWN_REF RunRef
 342 );
 343
 344 BOOLEAN
 345 FASTCALL
 346 ExfAcquireRundownProtectionEx(
 347      IN OUT PEX_RUNDOWN_REF RunRef,
 348      IN ULONG Count
 349 );
 350
 351 VOID
 352 FASTCALL
 353 ExfReleaseRundownProtection(
 354      IN OUT PEX_RUNDOWN_REF RunRef
 355 );
 356
 357 VOID
 358 FASTCALL
 359 ExfReleaseRundownProtectionEx(
 360      IN OUT PEX_RUNDOWN_REF RunRef,
 361      IN ULONG Count
 362 );
 363
 364 VOID
 365 FASTCALL
 366 ExfRundownCompleted(
 367      OUT PEX_RUNDOWN_REF RunRef
 368 );
 369
 370 VOID
 371 FASTCALL
 372 ExfWaitForRundownProtectionRelease(
 373      IN OUT PEX_RUNDOWN_REF RunRef
 374 );
 375
 376 /* HANDLE TABLE FUNCTIONS ***************************************************/
 377
 378 typedef BOOLEAN
 379 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 380     PHANDLE_TABLE_ENTRY HandleTableEntry,
 381     HANDLE Handle,
 382     PVOID Context
 383 );
 384
 385 typedef BOOLEAN
 386 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 387     IN PEPROCESS Process,
 388     IN PHANDLE_TABLE HandleTable,
 389     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 390     IN PHANDLE_TABLE_ENTRY NewEntry
 391 );
 392
 393 typedef BOOLEAN
 394 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 395     PHANDLE_TABLE_ENTRY HandleTableEntry,
 396     ULONG_PTR Context
 397 );
 398
 399 VOID
 400 NTAPI
 401 ExpInitializeHandleTables(
 402     VOID
 403 );
 404
 405 PHANDLE_TABLE
 406 NTAPI
 407 ExCreateHandleTable(
 408     IN PEPROCESS Process OPTIONAL
 409 );
 410
 411 VOID
 412 NTAPI
 413 ExUnlockHandleTableEntry(
 414     IN PHANDLE_TABLE HandleTable,
 415     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 416 );
 417
 418 HANDLE
 419 NTAPI
 420 ExCreateHandle(
 421     IN PHANDLE_TABLE HandleTable,
 422     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 423 );
 424
 425 VOID
 426 NTAPI
 427 ExDestroyHandleTable(
 428     IN PHANDLE_TABLE HandleTable,
 429     IN PVOID DestroyHandleProcedure OPTIONAL
 430 );
 431
 432 BOOLEAN
 433 NTAPI
 434 ExDestroyHandle(
 435     IN PHANDLE_TABLE HandleTable,
 436     IN HANDLE Handle,
 437     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 438 );
 439
 440 PHANDLE_TABLE_ENTRY
 441 NTAPI
 442 ExMapHandleToPointer(
 443     IN PHANDLE_TABLE HandleTable,
 444     IN HANDLE Handle
 445 );
 446
 447 PHANDLE_TABLE
 448 NTAPI
 449 ExDupHandleTable(
 450     IN PEPROCESS Process,
 451     IN PHANDLE_TABLE HandleTable,
 452     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 453     IN ULONG_PTR Mask
 454 );
 455
 456 BOOLEAN
 457 NTAPI
 458 ExChangeHandle(
 459     IN PHANDLE_TABLE HandleTable,
 460     IN HANDLE Handle,
 461     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 462     IN ULONG_PTR Context
 463 );
 464
 465 VOID
 466 NTAPI
 467 ExSweepHandleTable(
 468     IN PHANDLE_TABLE HandleTable,
 469     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 470     IN PVOID Context
 471 );
 472
 473 PHANDLE_TABLE_ENTRY
 474 NTAPI
 475 ExpLookupHandleTableEntry(
 476     IN PHANDLE_TABLE HandleTable,
 477     IN EXHANDLE Handle
 478 );
 479
 480 BOOLEAN
 481 NTAPI
 482 ExpLockHandleTableEntry(
 483     IN PHANDLE_TABLE HandleTable,
 484     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 485 );
 486
 487 /* PSEH EXCEPTION HANDLING **************************************************/
 488
 489 LONG
 490 NTAPI
 491 ExSystemExceptionFilter(VOID);
 492
 493 /* CALLBACKS *****************************************************************/
 494
 495 FORCEINLINE
 496 VOID
 497 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 498              IN PVOID Context,
 499              IN PVOID Argument1,
 500              IN PVOID Argument2)
 501 {
 502     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 503     PEX_CALLBACK_FUNCTION Function;
 504
 505     /* Reference the block */
 506     CallbackBlock = ExReferenceCallBackBlock(Callback);
 507     if (CallbackBlock)
 508     {
 509         /* Get the function */
 510         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 511
 512         /* Do the callback */
 513         Function(Context, Argument1, Argument2);
 514
 515         /* Now dereference it */
 516         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 517     }
 518 }
 519
 520 /* FAST REFS ******************************************************************/
 521
 522 FORCEINLINE
 523 PVOID
 524 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 525 {
 526     /* Return the unbiased pointer */
 527     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 528 }
 529
 530 FORCEINLINE
 531 ULONG
 532 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 533 {
 534     /* Return the reference count */
 535     return (ULONG)FastRef.RefCnt;
 536 }
 537
 538 FORCEINLINE
 539 VOID
 540 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 541                           IN OPTIONAL PVOID Object)
 542 {
 543     /* Sanity check */
 544     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 545
 546     /* Check if an object is being set */
 547     if (!Object)
 548     {
 549         /* Clear the field */
 550         FastRef->Object = NULL;
 551     }
 552     else
 553     {
 554         /* Otherwise, we assume the object was referenced and is ready */
 555         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 556     }
 557 }
 558
 559 FORCEINLINE
 560 EX_FAST_REF
 561 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 562 {
 563     EX_FAST_REF OldValue, NewValue;
 564
 565     /* Start reference loop */
 566     for (;;)
 567     {
 568         /* Get the current reference count */
 569         OldValue = *FastRef;
 570         if (OldValue.RefCnt)
 571         {
 572             /* Increase the reference count */
 573             NewValue.Value = OldValue.Value - 1;
 574             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 575                                                 NewValue.Object,
 576                                                 OldValue.Object);
 577             if (NewValue.Object != OldValue.Object) continue;
 578         }
 579
 580         /* We are done */
 581         break;
 582     }
 583
 584     /* Return the old value */
 585     return OldValue;
 586 }
 587
 588 FORCEINLINE
 589 BOOLEAN
 590 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 591                       IN PVOID Object)
 592 {
 593     EX_FAST_REF OldValue, NewValue;
 594
 595     /* Sanity checks */
 596     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 597
 598     /* Start update loop */
 599     for (;;)
 600     {
 601         /* Get the current reference count */
 602         OldValue = *FastRef;
 603
 604         /* Check if the current count is too high or if the pointer changed */
 605         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 606             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 607         {
 608             /* Fail */
 609             return FALSE;
 610         }
 611
 612         /* Update the reference count */
 613         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 614         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 615                                             NewValue.Object,
 616                                             OldValue.Object);
 617         if (NewValue.Object != OldValue.Object) continue;
 618
 619         /* We are done */
 620         break;
 621     }
 622
 623     /* Return success */
 624     return TRUE;
 625 }
 626
 627 FORCEINLINE
 628 BOOLEAN
 629 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 630                        IN PVOID Object)
 631 {
 632     EX_FAST_REF OldValue, NewValue;
 633
 634     /* Sanity checks */
 635     ASSERT(Object != NULL);
 636     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 637
 638     /* Start reference loop */
 639     for (;;)
 640     {
 641         /* Get the current reference count */
 642         OldValue = *FastRef;
 643
 644         /* Check if we're full if if the pointer changed */
 645         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 646
 647         /* Decrease the reference count */
 648         NewValue.Value = OldValue.Value + 1;
 649         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 650                                             NewValue.Object,
 651                                             OldValue.Object);
 652         if (NewValue.Object != OldValue.Object) continue;
 653
 654         /* We are done */
 655         break;
 656     }
 657
 658     /* Return success */
 659     return TRUE;
 660 }
 661
 662 FORCEINLINE
 663 EX_FAST_REF
 664 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 665                     IN PVOID Object)
 666 {
 667     EX_FAST_REF NewValue, OldValue;
 668
 669     /* Sanity check */
 670     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 671
 672     /* Check if an object is being set */
 673     if (!Object)
 674     {
 675         /* Clear the field */
 676         NewValue.Object = NULL;
 677     }
 678     else
 679     {
 680         /* Otherwise, we assume the object was referenced and is ready */
 681         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 682     }
 683
 684     /* Update the object */
 685     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 686     return OldValue;
 687 }
 688
 689 FORCEINLINE
 690 EX_FAST_REF
 691 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 692                            IN PVOID Object,
 693                            IN PVOID OldObject)
 694 {
 695     EX_FAST_REF OldValue, NewValue;
 696
 697     /* Sanity check and start swap loop */
 698     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 699     for (;;)
 700     {
 701         /* Get the current value */
 702         OldValue = *FastRef;
 703
 704         /* Make sure there's enough references to swap */
 705         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 706
 707         /* Check if we have an object to swap */
 708         if (Object)
 709         {
 710             /* Set up the value with maximum fast references */
 711             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 712         }
 713         else
 714         {
 715             /* Write the object address itself (which is empty) */
 716             NewValue.Value = (ULONG_PTR)Object;
 717         }
 718
 719         /* Do the actual compare exchange */
 720         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 721                                             NewValue.Object,
 722                                             OldValue.Object);
 723         if (NewValue.Object != OldValue.Object) continue;
 724
 725         /* All done */
 726         break;
 727     }
 728
 729     /* Return the old value */
 730     return OldValue;
 731 }
 732
 733 /* RUNDOWN *******************************************************************/
 734
 735 /*++
 736  * @name ExfAcquireRundownProtection
 737  * INTERNAL MACRO
 738  *
 739  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 740  *     the specified descriptor.
 741  *
 742  * @param RunRef
 743  *        Pointer to a rundown reference descriptor.
 744  *
 745  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 746  *
 747  * @remarks This is the internal macro for system use only.In case the rundown
 748  *          was active, then the slow-path will be called through the exported
 749  *          function.
 750  *
 751  *--*/
 752 FORCEINLINE
 753 BOOLEAN
 754 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 755 {
 756     ULONG_PTR Value, NewValue;
 757
 758     /* Get the current value and mask the active bit */
 759     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 760
 761     /* Add a reference */
 762     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 763
 764     /* Change the value */
 765     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 766     if (NewValue != Value)
 767     {
 768         /* Rundown was active, use long path */
 769         return ExfAcquireRundownProtection(RunRef);
 770     }
 771
 772     /* Success */
 773     return TRUE;
 774 }
 775
 776 /*++
 777  * @name ExReleaseRundownProtection
 778  * INTERNAL MACRO
 779  *
 780  *     The ExReleaseRundownProtection routine releases rundown protection for
 781  *     the specified descriptor.
 782  *
 783  * @param RunRef
 784  *        Pointer to a rundown reference descriptor.
 785  *
 786  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 787  *
 788  * @remarks This is the internal macro for system use only.In case the rundown
 789  *          was active, then the slow-path will be called through the exported
 790  *          function.
 791  *
 792  *--*/
 793 FORCEINLINE
 794 VOID
 795 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 796 {
 797     ULONG_PTR Value, NewValue;
 798
 799     /* Get the current value and mask the active bit */
 800     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 801
 802     /* Remove a reference */
 803     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 804
 805     /* Change the value */
 806     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 807
 808     /* Check if the rundown was active */
 809     if (NewValue != Value)
 810     {
 811         /* Rundown was active, use long path */
 812         ExfReleaseRundownProtection(RunRef);
 813     }
 814     else
 815     {
 816         /* Sanity check */
 817         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 818     }
 819 }
 820
 821 /*++
 822  * @name ExInitializeRundownProtection
 823  * INTERNAL MACRO
 824  *
 825  *     The ExInitializeRundownProtection routine initializes a rundown
 826  *     protection descriptor.
 827  *
 828  * @param RunRef
 829  *        Pointer to a rundown reference descriptor.
 830  *
 831  * @return None.
 832  *
 833  * @remarks This is the internal macro for system use only.
 834  *
 835  *--*/
 836 FORCEINLINE
 837 VOID
 838 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 839 {
 840     /* Set the count to zero */
 841     RunRef->Count = 0;
 842 }
 843
 844 /*++
 845  * @name ExWaitForRundownProtectionRelease
 846  * INTERNAL MACRO
 847  *
 848  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 849  *     rundown descriptor has been released.
 850  *
 851  * @param RunRef
 852  *        Pointer to a rundown reference descriptor.
 853  *
 854  * @return None.
 855  *
 856  * @remarks This is the internal macro for system use only. If a wait is actually
 857  *          necessary, then the slow path is taken through the exported function.
 858  *
 859  *--*/
 860 FORCEINLINE
 861 VOID
 862 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 863 {
 864     ULONG_PTR Value;
 865
 866     /* Set the active bit */
 867     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 868     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 869     {
 870         /* If the the rundown wasn't already active, then take the long path */
 871         ExfWaitForRundownProtectionRelease(RunRef);
 872     }
 873 }
 874
 875 /*++
 876  * @name ExRundownCompleted
 877  * INTERNAL MACRO
 878  *
 879  *     The ExRundownCompleted routine completes the rundown of the specified
 880  *     descriptor by setting the active bit.
 881  *
 882  * @param RunRef
 883  *        Pointer to a rundown reference descriptor.
 884  *
 885  * @return None.
 886  *
 887  * @remarks This is the internal macro for system use only.
 888  *
 889  *--*/
 890 FORCEINLINE
 891 VOID
 892 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 893 {
 894     /* Sanity check */
 895     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 896
 897     /* Mark the counter as active */
 898     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 899 }
 900
 901 /* PUSHLOCKS *****************************************************************/
 902
 903 /* FIXME: VERIFY THESE! */
 904
 905 VOID
 906 FASTCALL
 907 ExBlockPushLock(
 908     IN PEX_PUSH_LOCK PushLock,
 909     IN PVOID WaitBlock
 910 );
 911
 912 VOID
 913 FASTCALL
 914 ExfUnblockPushLock(
 915     IN PEX_PUSH_LOCK PushLock,
 916     IN PVOID CurrentWaitBlock
 917 );
 918
 919 VOID
 920 FASTCALL
 921 ExWaitForUnblockPushLock(
 922     IN PEX_PUSH_LOCK PushLock,
 923     IN PVOID WaitBlock
 924 );
 925
 926 /*++
 927  * @name _ExInitializePushLock
 928  * INTERNAL MACRO
 929  *
 930  *     The _ExInitializePushLock macro initializes a PushLock.
 931  *
 932  * @params PushLock
 933  *         Pointer to the pushlock which is to be initialized.
 934  *
 935  * @return None.
 936  *
 937  * @remarks None.
 938  *
 939  *--*/
 940 FORCEINLINE
 941 VOID
 942 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 943 {
 944     /* Set the value to 0 */
 945     PushLock->Ptr = 0;
 946 }
 947 #define ExInitializePushLock _ExInitializePushLock
 948
 949 /*++
 950  * @name ExAcquirePushLockExclusive
 951  * INTERNAL MACRO
 952  *
 953  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 954  *
 955  * @params PushLock
 956  *         Pointer to the pushlock which is to be acquired.
 957  *
 958  * @return None.
 959  *
 960  * @remarks The function attempts the quickest route to acquire the lock, which is
 961  *          to simply set the lock bit.
 962  *          However, if the pushlock is already shared, the slower path is taken.
 963  *
 964  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 965  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 966  *
 967  *--*/
 968 FORCEINLINE
 969 VOID
 970 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 971 {
 972     /* Try acquiring the lock */
 973     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 974     {
 975         /* Someone changed it, use the slow path */
 976         ExfAcquirePushLockExclusive(PushLock);
 977     }
 978
 979     /* Sanity check */
 980     ASSERT(PushLock->Locked);
 981 }
 982
 983 /*++
 984 * @name ExTryToAcquirePushLockExclusive
 985 * INTERNAL MACRO
 986 *
 987 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 988 *
 989 * @params PushLock
 990 *         Pointer to the pushlock which is to be acquired.
 991 *
 992 * @return None.
 993 *
 994 * @remarks The function attempts the quickest route to acquire the lock, which is
 995 *          to simply set the lock bit.
 996 *          However, if the pushlock is already shared, the slower path is taken.
 997 *
 998 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 999 *          This macro should usually be paired up with KeAcquireCriticalRegion.
1000 *
1001 *--*/
1002 FORCEINLINE
1003 BOOLEAN
1004 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1005 {
1006     /* Try acquiring the lock */
1007     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1008     {
1009         /* Can't acquire */
1010         return FALSE;
1011     }
1012
1013     /* Got acquired */
1014     ASSERT (PushLock->Locked);
1015     return TRUE;
1016 }
1017
1018 /*++
1019  * @name ExAcquirePushLockShared
1020  * INTERNAL MACRO
1021  *
1022  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1023  *
1024  * @params PushLock
1025  *         Pointer to the pushlock which is to be acquired.
1026  *
1027  * @return None.
1028  *
1029  * @remarks The function attempts the quickest route to acquire the lock, which is
1030  *          to simply set the lock bit and set the share count to one.
1031  *          However, if the pushlock is already shared, the slower path is taken.
1032  *
1033  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1034  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1035  *
1036  *--*/
1037 FORCEINLINE
1038 VOID
1039 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1040 {
1041     EX_PUSH_LOCK NewValue;
1042
1043     /* Try acquiring the lock */
1044     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1045     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1046     {
1047         /* Someone changed it, use the slow path */
1048         ExfAcquirePushLockShared(PushLock);
1049     }
1050
1051     /* Sanity checks */
1052     ASSERT(PushLock->Locked);
1053 }
1054
1055 /*++
1056  * @name ExConvertPushLockSharedToExclusive
1057  * INTERNAL MACRO
1058  *
1059  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1060  *     pushlock to a shared pushlock.
1061  *
1062  * @params PushLock
1063  *         Pointer to the pushlock which is to be converted.
1064  *
1065  * @return FALSE if conversion failed, TRUE otherwise.
1066  *
1067  * @remarks The function attempts the quickest route to convert the lock, which is
1068  *          to simply set the lock bit and remove any other bits.
1069  *
1070  *--*/
1071 FORCEINLINE
1072 BOOLEAN
1073 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1074 {
1075     EX_PUSH_LOCK OldValue;
1076
1077     /* Set the expected old value */
1078     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1079
1080     /* Try converting the lock */
1081     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1082         OldValue.Ptr)
1083     {
1084         /* Conversion failed */
1085         return FALSE;
1086     }
1087
1088     /* Sanity check */
1089     ASSERT(PushLock->Locked);
1090     return TRUE;
1091 }
1092
1093 /*++
1094  * @name ExWaitOnPushLock
1095  * INTERNAL MACRO
1096  *
1097  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1098  *
1099  * @params PushLock
1100  *         Pointer to a pushlock.
1101  *
1102  * @return None.
1103  *
1104  * @remarks The function attempts to get any exclusive waiters out of their slow
1105  *          path by forcing an instant acquire/release operation.
1106  *
1107  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1108  *
1109  *--*/
1110 FORCEINLINE
1111 VOID
1112 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1113 {
1114     /* Check if we're locked */
1115     if (PushLock->Locked)
1116     {
1117         /* Acquire the lock */
1118         ExfAcquirePushLockExclusive(PushLock);
1119         ASSERT(PushLock->Locked);
1120
1121         /* Release it */
1122         ExfReleasePushLockExclusive(PushLock);
1123     }
1124 }
1125
1126 /*++
1127  * @name ExReleasePushLockShared
1128  * INTERNAL MACRO
1129  *
1130  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1131  *
1132  * @params PushLock
1133  *         Pointer to a previously acquired pushlock.
1134  *
1135  * @return None.
1136  *
1137  * @remarks The function attempts the quickest route to release the lock, which is
1138  *          to simply decrease the share count and remove the lock bit.
1139  *          However, if the pushlock is being waited on then the long path is taken.
1140  *
1141  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1142  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1143  *
1144  *--*/
1145 FORCEINLINE
1146 VOID
1147 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1148 {
1149     EX_PUSH_LOCK OldValue;
1150
1151     /* Sanity checks */
1152     ASSERT(PushLock->Locked);
1153
1154     /* Try to clear the pushlock */
1155     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1156     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1157     {
1158         /* There are still other people waiting on it */
1159         ExfReleasePushLockShared(PushLock);
1160     }
1161 }
1162
1163 /*++
1164  * @name ExReleasePushLockExclusive
1165  * INTERNAL MACRO
1166  *
1167  *     The ExReleasePushLockExclusive macro releases a previously
1168  *     exclusively acquired PushLock.
1169  *
1170  * @params PushLock
1171  *         Pointer to a previously acquired pushlock.
1172  *
1173  * @return None.
1174  *
1175  * @remarks The function attempts the quickest route to release the lock, which is
1176  *          to simply clear the locked bit.
1177  *          However, if the pushlock is being waited on, the slow path is taken
1178  *          in an attempt to wake up the lock.
1179  *
1180  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1181  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1182  *
1183  *--*/
1184 FORCEINLINE
1185 VOID
1186 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1187 {
1188     EX_PUSH_LOCK OldValue;
1189
1190     /* Sanity checks */
1191     ASSERT(PushLock->Locked);
1192
1193     /* Unlock the pushlock */
1194     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1195                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1196
1197     /* Sanity checks */
1198     ASSERT(OldValue.Locked);
1199     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1200
1201     /* Check if anyone is waiting on it and it's not already waking*/
1202     if ((OldValue.Waiting) && !(OldValue.Waking))
1203     {
1204         /* Wake it up */
1205         ExfTryToWakePushLock(PushLock);
1206     }
1207 }
1208
1209 /*++
1210  * @name ExReleasePushLock
1211  * INTERNAL MACRO
1212  *
1213  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1214  *
1215  * @params PushLock
1216  *         Pointer to a previously acquired pushlock.
1217  *
1218  * @return None.
1219  *
1220  * @remarks The function attempts the quickest route to release the lock, which is
1221  *          to simply clear all the fields and decrease the share count if required.
1222  *          However, if the pushlock is being waited on then the long path is taken.
1223  *
1224  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1225  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1226  *
1227  *--*/
1228 FORCEINLINE
1229 VOID
1230 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1231 {
1232     EX_PUSH_LOCK OldValue = *PushLock;
1233     EX_PUSH_LOCK NewValue;
1234
1235     /* Sanity checks */
1236     ASSERT(OldValue.Locked);
1237
1238     /* Check if the pushlock is shared */
1239     if (OldValue.Shared > 1)
1240     {
1241         /* Decrease the share count */
1242         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1243     }
1244     else
1245     {
1246         /* Clear the pushlock entirely */
1247         NewValue.Value = 0;
1248     }
1249
1250     /* Check if nobody is waiting on us and try clearing the lock here */
1251     if ((OldValue.Waiting) ||
1252         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1253          OldValue.Ptr))
1254     {
1255         /* We have waiters, use the long path */
1256         ExfReleasePushLock(PushLock);
1257     }
1258 }
1259
1260 /* FAST MUTEX INLINES *********************************************************/
1261
1262 FORCEINLINE
1263 VOID
1264 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1265 {
1266     PKTHREAD Thread = KeGetCurrentThread();
1267
1268     /* Sanity check */
1269     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1270            (Thread->CombinedApcDisable != 0) ||
1271            (Thread->Teb == NULL) ||
1272            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1273     ASSERT(FastMutex->Owner != Thread);
1274
1275     /* Decrease the count */
1276     if (InterlockedDecrement(&FastMutex->Count))
1277     {
1278         /* Someone is still holding it, use slow path */
1279         KiAcquireFastMutex(FastMutex);
1280     }
1281
1282     /* Set the owner */
1283     FastMutex->Owner = Thread;
1284 }
1285
1286 FORCEINLINE
1287 VOID
1288 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1289 {
1290     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1291            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1292            (KeGetCurrentThread()->Teb == NULL) ||
1293            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1294     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1295
1296     /* Erase the owner */
1297     FastMutex->Owner = NULL;
1298
1299     /* Increase the count */
1300     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1301     {
1302         /* Someone was waiting for it, signal the waiter */
1303         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1304     }
1305 }
1306
1307 FORCEINLINE
1308 VOID
1309 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1310 {
1311     KIRQL OldIrql;
1312     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1313
1314     /* Raise IRQL to APC */
1315     KeRaiseIrql(APC_LEVEL, &OldIrql);
1316
1317     /* Decrease the count */
1318     if (InterlockedDecrement(&FastMutex->Count))
1319     {
1320         /* Someone is still holding it, use slow path */
1321         KiAcquireFastMutex(FastMutex);
1322     }
1323
1324     /* Set the owner and IRQL */
1325     FastMutex->Owner = KeGetCurrentThread();
1326     FastMutex->OldIrql = OldIrql;
1327 }
1328
1329 FORCEINLINE
1330 VOID
1331 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1332 {
1333     KIRQL OldIrql;
1334     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1335
1336     /* Erase the owner */
1337     FastMutex->Owner = NULL;
1338     OldIrql = (KIRQL)FastMutex->OldIrql;
1339
1340     /* Increase the count */
1341     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1342     {
1343         /* Someone was waiting for it, signal the waiter */
1344         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1345     }
1346
1347     /* Lower IRQL back */
1348     KeLowerIrql(OldIrql);
1349 }
1350
1351 FORCEINLINE
1352 BOOLEAN
1353 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1354 {
1355     KIRQL OldIrql;
1356     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1357
1358     /* Raise to APC_LEVEL */
1359     KeRaiseIrql(APC_LEVEL, &OldIrql);
1360
1361     /* Check if we can quickly acquire it */
1362     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1363     {
1364         /* We have, set us as owners */
1365         FastMutex->Owner = KeGetCurrentThread();
1366         FastMutex->OldIrql = OldIrql;
1367         return TRUE;
1368     }
1369     else
1370     {
1371         /* Acquire attempt failed */
1372         KeLowerIrql(OldIrql);
1373         YieldProcessor();
1374         return FALSE;
1375     }
1376 }
1377
1378 FORCEINLINE
1379 VOID
1380 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1381 {
1382     /* Enter the Critical Region */
1383     KeEnterCriticalRegion();
1384
1385     /* Acquire the mutex unsafely */
1386     _ExAcquireFastMutexUnsafe(FastMutex);
1387 }
1388
1389 FORCEINLINE
1390 VOID
1391 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1392 {
1393     /* Release the mutex unsafely */
1394     _ExReleaseFastMutexUnsafe(FastMutex);
1395
1396     /* Leave the critical region */
1397     KeLeaveCriticalRegion();
1398 }
1399
1400 /* OTHER FUNCTIONS **********************************************************/
1401
1402 BOOLEAN
1403 NTAPI
1404 ExTryToAcquireResourceExclusiveLite(
1405     IN PERESOURCE Resource
1406 );
1407
1408 NTSTATUS
1409 ExpSetTimeZoneInformation(
1410     IN PTIME_ZONE_INFORMATION TimeZoneInformation
1411 );
1412
1413 BOOLEAN
1414 NTAPI
1415 ExAcquireTimeRefreshLock(
1416     IN BOOLEAN Wait
1417 );
1418
1419 VOID
1420 NTAPI
1421 ExReleaseTimeRefreshLock(
1422     VOID
1423 );
1424
1425 VOID
1426 NTAPI
1427 ExUpdateSystemTimeFromCmos(
1428     IN BOOLEAN UpdateInterruptTime,
1429     IN ULONG MaxSepInSeconds
1430 );
1431
1432 VOID
1433 NTAPI
1434 ExAllocateLocallyUniqueId(
1435     OUT LUID *LocallyUniqueId
1436 );
1437
1438 VOID
1439 NTAPI
1440 ExTimerRundown(
1441     VOID
1442 );
1443
1444 VOID
1445 NTAPI
1446 HeadlessInit(
1447     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1448 );
1449
1450 VOID
1451 NTAPI
1452 XIPInit(
1453     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1454 );
1455
1456 #define InterlockedDecrementUL(Addend) \
1457    (ULONG)InterlockedDecrement((PLONG)(Addend))
1458
1459 #define InterlockedIncrementUL(Addend) \
1460    (ULONG)InterlockedIncrement((PLONG)(Addend))
1461
1462 #define InterlockedExchangeUL(Target, Value) \
1463    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1464
1465 #define InterlockedExchangeAddUL(Addend, Value) \
1466    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1467
1468 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1469    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1470
1471 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1472    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))