ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31 extern ULONG ExCriticalWorkerThreads;
  32 extern ULONG ExDelayedWorkerThreads;
  33
  34 extern PVOID ExpDefaultErrorPort;
  35 extern PEPROCESS ExpDefaultErrorPortProcess;
  36
  37 /*
  38  * NT/Cm Version Info variables
  39  */
  40 extern ULONG NtMajorVersion;
  41 extern ULONG NtMinorVersion;
  42 extern ULONG NtBuildNumber;
  43 extern ULONG CmNtSpBuildNumber;
  44 extern ULONG CmNtCSDVersion;
  45 extern ULONG CmNtCSDReleaseType;
  46 extern UNICODE_STRING CmVersionString;
  47 extern UNICODE_STRING CmCSDVersionString;
  48 extern CHAR NtBuildLab[];
  49
  50 // #ifdef _WINKD_
  51 /*
  52  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  53  */
  54 typedef enum _WINKD_WORKER_STATE
  55 {
  56     WinKdWorkerReady = 0,
  57     WinKdWorkerStart,
  58     WinKdWorkerInitialized
  59 } WINKD_WORKER_STATE;
  60
  61 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  62 extern WINKD_WORKER_STATE ExpDebuggerWork;
  63 extern PEPROCESS ExpDebuggerProcessAttach;
  64 extern PEPROCESS ExpDebuggerProcessKill;
  65 extern ULONG_PTR ExpDebuggerPageIn;
  66
  67 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  68 // #endif /* _WINKD_ */
  69
  70 #ifdef _WIN64
  71 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  72 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  73 #else
  74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  76 #endif
  77 #define HANDLE_TAG_BITS (2)
  78 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
  79 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
  80
  81 typedef union _EXHANDLE
  82 {
  83      struct
  84      {
  85          ULONG_PTR TagBits:     HANDLE_TAG_BITS;
  86          ULONG_PTR Index:       HANDLE_INDEX_BITS;
  87          ULONG_PTR KernelFlag : KERNEL_FLAG_BITS;
  88      };
  89      struct
  90      {
  91          ULONG_PTR TagBits2:    HANDLE_TAG_BITS;
  92          ULONG_PTR LowIndex:    HANDLE_LOW_BITS;
  93          ULONG_PTR MidIndex:    HANDLE_HIGH_BITS;
  94          ULONG_PTR HighIndex:   HANDLE_HIGH_BITS;
  95          ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
  96      };
  97      HANDLE GenericHandleOverlay;
  98      ULONG_PTR Value;
  99      ULONG AsULONG;
 100 } EXHANDLE, *PEXHANDLE;
 101
 102 typedef struct _ETIMER
 103 {
 104     KTIMER KeTimer;
 105     KAPC TimerApc;
 106     KDPC TimerDpc;
 107     LIST_ENTRY ActiveTimerListEntry;
 108     KSPIN_LOCK Lock;
 109     LONG Period;
 110     BOOLEAN ApcAssociated;
 111     BOOLEAN WakeTimer;
 112     LIST_ENTRY WakeTimerListEntry;
 113 } ETIMER, *PETIMER;
 114
 115 typedef struct
 116 {
 117     PCALLBACK_OBJECT *CallbackObject;
 118     PWSTR Name;
 119 } SYSTEM_CALLBACKS;
 120
 121 typedef struct _HARDERROR_USER_PARAMETERS
 122 {
 123     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 124     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 125     WCHAR Buffer[ANYSIZE_ARRAY];
 126 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 127
 128 #define MAX_FAST_REFS           7
 129
 130 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 131 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 132 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 133 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 134 #define ExRundownCompleted                              _ExRundownCompleted
 135 #define ExGetPreviousMode                               KeGetPreviousMode
 136
 137
 138 //
 139 // Various bits tagged on the handle or handle table
 140 //
 141 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 142 #define FREE_HANDLE_MASK                -1
 143
 144 //
 145 // Number of entries in each table level
 146 //
 147 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 148 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 149 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 150
 151 //
 152 // Maximum index in each table level before we need another table
 153 //
 154 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 155 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 156 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 157
 158 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 159 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 160 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 161
 162 NTSTATUS
 163 NTAPI
 164 ExGetPoolTagInfo(
 165     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 166     IN ULONG SystemInformationLength,
 167     IN OUT PULONG ReturnLength OPTIONAL
 168 );
 169
 170 /* INITIALIZATION FUNCTIONS *************************************************/
 171
 172 BOOLEAN
 173 NTAPI
 174 ExpWin32kInit(VOID);
 175
 176 VOID
 177 NTAPI
 178 ExInit2(VOID);
 179
 180 VOID
 181 NTAPI
 182 Phase1Initialization(
 183     IN PVOID Context
 184 );
 185
 186 VOID
 187 NTAPI
 188 ExpInitializePushLocks(VOID);
 189
 190 BOOLEAN
 191 NTAPI
 192 ExRefreshTimeZoneInformation(
 193     IN PLARGE_INTEGER SystemBootTime
 194 );
 195
 196 VOID
 197 NTAPI
 198 ExpInitializeWorkerThreads(VOID);
 199
 200 VOID
 201 NTAPI
 202 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 203
 204 VOID
 205 NTAPI
 206 ExpInitLookasideLists(VOID);
 207
 208 VOID
 209 NTAPI
 210 ExInitializeSystemLookasideList(
 211     IN PGENERAL_LOOKASIDE List,
 212     IN POOL_TYPE Type,
 213     IN ULONG Size,
 214     IN ULONG Tag,
 215     IN USHORT MaximumDepth,
 216     IN PLIST_ENTRY ListHead
 217 );
 218
 219 BOOLEAN
 220 NTAPI
 221 ExpInitializeCallbacks(VOID);
 222
 223 VOID
 224 NTAPI
 225 ExpInitUuids(VOID);
 226
 227 VOID
 228 NTAPI
 229 ExpInitializeExecutive(
 230     IN ULONG Cpu,
 231     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 232 );
 233
 234 VOID
 235 NTAPI
 236 ExShutdownSystem(VOID);
 237
 238 BOOLEAN
 239 NTAPI
 240 ExpInitializeEventImplementation(VOID);
 241
 242 BOOLEAN
 243 NTAPI
 244 ExpInitializeKeyedEventImplementation(VOID);
 245
 246 BOOLEAN
 247 NTAPI
 248 ExpInitializeEventPairImplementation(VOID);
 249
 250 BOOLEAN
 251 NTAPI
 252 ExpInitializeSemaphoreImplementation(VOID);
 253
 254 BOOLEAN
 255 NTAPI
 256 ExpInitializeMutantImplementation(VOID);
 257
 258 BOOLEAN
 259 NTAPI
 260 ExpInitializeTimerImplementation(VOID);
 261
 262 BOOLEAN
 263 NTAPI
 264 ExpInitializeProfileImplementation(VOID);
 265
 266 VOID
 267 NTAPI
 268 ExpResourceInitialization(VOID);
 269
 270 VOID
 271 NTAPI
 272 ExInitPoolLookasidePointers(VOID);
 273
 274 /* Callback Functions ********************************************************/
 275
 276 VOID
 277 NTAPI
 278 ExInitializeCallBack(
 279     IN OUT PEX_CALLBACK Callback
 280 );
 281
 282 PEX_CALLBACK_ROUTINE_BLOCK
 283 NTAPI
 284 ExAllocateCallBack(
 285     IN PEX_CALLBACK_FUNCTION Function,
 286     IN PVOID Context
 287 );
 288
 289 VOID
 290 NTAPI
 291 ExFreeCallBack(
 292     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 293 );
 294
 295 BOOLEAN
 296 NTAPI
 297 ExCompareExchangeCallBack (
 298     IN OUT PEX_CALLBACK CallBack,
 299     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 300     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 301 );
 302
 303 PEX_CALLBACK_ROUTINE_BLOCK
 304 NTAPI
 305 ExReferenceCallBackBlock(
 306     IN OUT PEX_CALLBACK CallBack
 307 );
 308
 309 VOID
 310 NTAPI
 311 ExDereferenceCallBackBlock(
 312     IN OUT PEX_CALLBACK CallBack,
 313     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 314 );
 315
 316 PEX_CALLBACK_FUNCTION
 317 NTAPI
 318 ExGetCallBackBlockRoutine(
 319     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 320 );
 321
 322 PVOID
 323 NTAPI
 324 ExGetCallBackBlockContext(
 325     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 326 );
 327
 328 VOID
 329 NTAPI
 330 ExWaitForCallBacks(
 331     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 332 );
 333
 334 /* Rundown Functions ********************************************************/
 335
 336 VOID
 337 FASTCALL
 338 ExfInitializeRundownProtection(
 339      OUT PEX_RUNDOWN_REF RunRef
 340 );
 341
 342 VOID
 343 FASTCALL
 344 ExfReInitializeRundownProtection(
 345      OUT PEX_RUNDOWN_REF RunRef
 346 );
 347
 348 BOOLEAN
 349 FASTCALL
 350 ExfAcquireRundownProtection(
 351      IN OUT PEX_RUNDOWN_REF RunRef
 352 );
 353
 354 BOOLEAN
 355 FASTCALL
 356 ExfAcquireRundownProtectionEx(
 357      IN OUT PEX_RUNDOWN_REF RunRef,
 358      IN ULONG Count
 359 );
 360
 361 VOID
 362 FASTCALL
 363 ExfReleaseRundownProtection(
 364      IN OUT PEX_RUNDOWN_REF RunRef
 365 );
 366
 367 VOID
 368 FASTCALL
 369 ExfReleaseRundownProtectionEx(
 370      IN OUT PEX_RUNDOWN_REF RunRef,
 371      IN ULONG Count
 372 );
 373
 374 VOID
 375 FASTCALL
 376 ExfRundownCompleted(
 377      OUT PEX_RUNDOWN_REF RunRef
 378 );
 379
 380 VOID
 381 FASTCALL
 382 ExfWaitForRundownProtectionRelease(
 383      IN OUT PEX_RUNDOWN_REF RunRef
 384 );
 385
 386 /* HANDLE TABLE FUNCTIONS ***************************************************/
 387
 388 typedef BOOLEAN
 389 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 390     PHANDLE_TABLE_ENTRY HandleTableEntry,
 391     HANDLE Handle,
 392     PVOID Context
 393 );
 394
 395 typedef BOOLEAN
 396 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 397     IN PEPROCESS Process,
 398     IN PHANDLE_TABLE HandleTable,
 399     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 400     IN PHANDLE_TABLE_ENTRY NewEntry
 401 );
 402
 403 typedef BOOLEAN
 404 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 405     PHANDLE_TABLE_ENTRY HandleTableEntry,
 406     ULONG_PTR Context
 407 );
 408
 409 VOID
 410 NTAPI
 411 ExpInitializeHandleTables(
 412     VOID
 413 );
 414
 415 PHANDLE_TABLE
 416 NTAPI
 417 ExCreateHandleTable(
 418     IN PEPROCESS Process OPTIONAL
 419 );
 420
 421 VOID
 422 NTAPI
 423 ExUnlockHandleTableEntry(
 424     IN PHANDLE_TABLE HandleTable,
 425     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 426 );
 427
 428 HANDLE
 429 NTAPI
 430 ExCreateHandle(
 431     IN PHANDLE_TABLE HandleTable,
 432     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 433 );
 434
 435 VOID
 436 NTAPI
 437 ExDestroyHandleTable(
 438     IN PHANDLE_TABLE HandleTable,
 439     IN PVOID DestroyHandleProcedure OPTIONAL
 440 );
 441
 442 BOOLEAN
 443 NTAPI
 444 ExDestroyHandle(
 445     IN PHANDLE_TABLE HandleTable,
 446     IN HANDLE Handle,
 447     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 448 );
 449
 450 PHANDLE_TABLE_ENTRY
 451 NTAPI
 452 ExMapHandleToPointer(
 453     IN PHANDLE_TABLE HandleTable,
 454     IN HANDLE Handle
 455 );
 456
 457 PHANDLE_TABLE
 458 NTAPI
 459 ExDupHandleTable(
 460     IN PEPROCESS Process,
 461     IN PHANDLE_TABLE HandleTable,
 462     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 463     IN ULONG_PTR Mask
 464 );
 465
 466 BOOLEAN
 467 NTAPI
 468 ExChangeHandle(
 469     IN PHANDLE_TABLE HandleTable,
 470     IN HANDLE Handle,
 471     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 472     IN ULONG_PTR Context
 473 );
 474
 475 VOID
 476 NTAPI
 477 ExSweepHandleTable(
 478     IN PHANDLE_TABLE HandleTable,
 479     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 480     IN PVOID Context
 481 );
 482
 483 PHANDLE_TABLE_ENTRY
 484 NTAPI
 485 ExpLookupHandleTableEntry(
 486     IN PHANDLE_TABLE HandleTable,
 487     IN EXHANDLE Handle
 488 );
 489
 490 BOOLEAN
 491 NTAPI
 492 ExpLockHandleTableEntry(
 493     IN PHANDLE_TABLE HandleTable,
 494     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 495 );
 496
 497 /* PSEH EXCEPTION HANDLING **************************************************/
 498
 499 LONG
 500 NTAPI
 501 ExSystemExceptionFilter(VOID);
 502
 503 /* CALLBACKS *****************************************************************/
 504
 505 FORCEINLINE
 506 VOID
 507 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 508              IN PVOID Context,
 509              IN PVOID Argument1,
 510              IN PVOID Argument2)
 511 {
 512     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 513     PEX_CALLBACK_FUNCTION Function;
 514
 515     /* Reference the block */
 516     CallbackBlock = ExReferenceCallBackBlock(Callback);
 517     if (CallbackBlock)
 518     {
 519         /* Get the function */
 520         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 521
 522         /* Do the callback */
 523         Function(Context, Argument1, Argument2);
 524
 525         /* Now dereference it */
 526         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 527     }
 528 }
 529
 530 /* FAST REFS ******************************************************************/
 531
 532 FORCEINLINE
 533 PVOID
 534 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 535 {
 536     /* Return the unbiased pointer */
 537     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 538 }
 539
 540 FORCEINLINE
 541 ULONG
 542 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 543 {
 544     /* Return the reference count */
 545     return (ULONG)FastRef.RefCnt;
 546 }
 547
 548 FORCEINLINE
 549 PEX_RUNDOWN_REF
 550 ExGetRunRefForCurrentProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
 551 {
 552     return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 553                              RunRefCacheAware->RunRefSize *
 554                              (KeGetCurrentProcessorNumber() % RunRefCacheAware->Number));
 555 }
 556
 557 FORCEINLINE
 558 VOID
 559 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 560                           IN OPTIONAL PVOID Object)
 561 {
 562     /* Sanity check */
 563     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 564
 565     /* Check if an object is being set */
 566     if (!Object)
 567     {
 568         /* Clear the field */
 569         FastRef->Object = NULL;
 570     }
 571     else
 572     {
 573         /* Otherwise, we assume the object was referenced and is ready */
 574         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 575     }
 576 }
 577
 578 FORCEINLINE
 579 EX_FAST_REF
 580 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 581 {
 582     EX_FAST_REF OldValue, NewValue;
 583
 584     /* Start reference loop */
 585     for (;;)
 586     {
 587         /* Get the current reference count */
 588         OldValue = *FastRef;
 589         if (OldValue.RefCnt)
 590         {
 591             /* Increase the reference count */
 592             NewValue.Value = OldValue.Value - 1;
 593             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 594                                                 NewValue.Object,
 595                                                 OldValue.Object);
 596             if (NewValue.Object != OldValue.Object) continue;
 597         }
 598
 599         /* We are done */
 600         break;
 601     }
 602
 603     /* Return the old value */
 604     return OldValue;
 605 }
 606
 607 FORCEINLINE
 608 BOOLEAN
 609 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 610                       IN PVOID Object)
 611 {
 612     EX_FAST_REF OldValue, NewValue;
 613
 614     /* Sanity checks */
 615     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 616
 617     /* Start update loop */
 618     for (;;)
 619     {
 620         /* Get the current reference count */
 621         OldValue = *FastRef;
 622
 623         /* Check if the current count is too high or if the pointer changed */
 624         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 625             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 626         {
 627             /* Fail */
 628             return FALSE;
 629         }
 630
 631         /* Update the reference count */
 632         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 633         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 634                                             NewValue.Object,
 635                                             OldValue.Object);
 636         if (NewValue.Object != OldValue.Object) continue;
 637
 638         /* We are done */
 639         break;
 640     }
 641
 642     /* Return success */
 643     return TRUE;
 644 }
 645
 646 FORCEINLINE
 647 BOOLEAN
 648 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 649                        IN PVOID Object)
 650 {
 651     EX_FAST_REF OldValue, NewValue;
 652
 653     /* Sanity checks */
 654     ASSERT(Object != NULL);
 655     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 656
 657     /* Start reference loop */
 658     for (;;)
 659     {
 660         /* Get the current reference count */
 661         OldValue = *FastRef;
 662
 663         /* Check if we're full if if the pointer changed */
 664         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 665
 666         /* Decrease the reference count */
 667         NewValue.Value = OldValue.Value + 1;
 668         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 669                                             NewValue.Object,
 670                                             OldValue.Object);
 671         if (NewValue.Object != OldValue.Object) continue;
 672
 673         /* We are done */
 674         break;
 675     }
 676
 677     /* Return success */
 678     return TRUE;
 679 }
 680
 681 FORCEINLINE
 682 EX_FAST_REF
 683 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 684                     IN PVOID Object)
 685 {
 686     EX_FAST_REF NewValue, OldValue;
 687
 688     /* Sanity check */
 689     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 690
 691     /* Check if an object is being set */
 692     if (!Object)
 693     {
 694         /* Clear the field */
 695         NewValue.Object = NULL;
 696     }
 697     else
 698     {
 699         /* Otherwise, we assume the object was referenced and is ready */
 700         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 701     }
 702
 703     /* Update the object */
 704     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 705     return OldValue;
 706 }
 707
 708 FORCEINLINE
 709 EX_FAST_REF
 710 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 711                            IN PVOID Object,
 712                            IN PVOID OldObject)
 713 {
 714     EX_FAST_REF OldValue, NewValue;
 715
 716     /* Sanity check and start swap loop */
 717     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 718     for (;;)
 719     {
 720         /* Get the current value */
 721         OldValue = *FastRef;
 722
 723         /* Make sure there's enough references to swap */
 724         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 725
 726         /* Check if we have an object to swap */
 727         if (Object)
 728         {
 729             /* Set up the value with maximum fast references */
 730             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 731         }
 732         else
 733         {
 734             /* Write the object address itself (which is empty) */
 735             NewValue.Value = (ULONG_PTR)Object;
 736         }
 737
 738         /* Do the actual compare exchange */
 739         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 740                                             NewValue.Object,
 741                                             OldValue.Object);
 742         if (NewValue.Object != OldValue.Object) continue;
 743
 744         /* All done */
 745         break;
 746     }
 747
 748     /* Return the old value */
 749     return OldValue;
 750 }
 751
 752 /* RUNDOWN *******************************************************************/
 753
 754 /*++
 755  * @name ExfAcquireRundownProtection
 756  * INTERNAL MACRO
 757  *
 758  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 759  *     the specified descriptor.
 760  *
 761  * @param RunRef
 762  *        Pointer to a rundown reference descriptor.
 763  *
 764  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 765  *
 766  * @remarks This is the internal macro for system use only.In case the rundown
 767  *          was active, then the slow-path will be called through the exported
 768  *          function.
 769  *
 770  *--*/
 771 FORCEINLINE
 772 BOOLEAN
 773 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 774 {
 775     ULONG_PTR Value, NewValue;
 776
 777     /* Get the current value and mask the active bit */
 778     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 779
 780     /* Add a reference */
 781     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 782
 783     /* Change the value */
 784     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 785     if (NewValue != Value)
 786     {
 787         /* Rundown was active, use long path */
 788         return ExfAcquireRundownProtection(RunRef);
 789     }
 790
 791     /* Success */
 792     return TRUE;
 793 }
 794
 795 /*++
 796  * @name ExReleaseRundownProtection
 797  * INTERNAL MACRO
 798  *
 799  *     The ExReleaseRundownProtection routine releases rundown protection for
 800  *     the specified descriptor.
 801  *
 802  * @param RunRef
 803  *        Pointer to a rundown reference descriptor.
 804  *
 805  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 806  *
 807  * @remarks This is the internal macro for system use only.In case the rundown
 808  *          was active, then the slow-path will be called through the exported
 809  *          function.
 810  *
 811  *--*/
 812 FORCEINLINE
 813 VOID
 814 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 815 {
 816     ULONG_PTR Value, NewValue;
 817
 818     /* Get the current value and mask the active bit */
 819     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 820
 821     /* Remove a reference */
 822     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 823
 824     /* Change the value */
 825     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 826
 827     /* Check if the rundown was active */
 828     if (NewValue != Value)
 829     {
 830         /* Rundown was active, use long path */
 831         ExfReleaseRundownProtection(RunRef);
 832     }
 833     else
 834     {
 835         /* Sanity check */
 836         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 837     }
 838 }
 839
 840 /*++
 841  * @name ExInitializeRundownProtection
 842  * INTERNAL MACRO
 843  *
 844  *     The ExInitializeRundownProtection routine initializes a rundown
 845  *     protection descriptor.
 846  *
 847  * @param RunRef
 848  *        Pointer to a rundown reference descriptor.
 849  *
 850  * @return None.
 851  *
 852  * @remarks This is the internal macro for system use only.
 853  *
 854  *--*/
 855 FORCEINLINE
 856 VOID
 857 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 858 {
 859     /* Set the count to zero */
 860     RunRef->Count = 0;
 861 }
 862
 863 /*++
 864  * @name ExWaitForRundownProtectionRelease
 865  * INTERNAL MACRO
 866  *
 867  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 868  *     rundown descriptor has been released.
 869  *
 870  * @param RunRef
 871  *        Pointer to a rundown reference descriptor.
 872  *
 873  * @return None.
 874  *
 875  * @remarks This is the internal macro for system use only. If a wait is actually
 876  *          necessary, then the slow path is taken through the exported function.
 877  *
 878  *--*/
 879 FORCEINLINE
 880 VOID
 881 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 882 {
 883     ULONG_PTR Value;
 884
 885     /* Set the active bit */
 886     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 887     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 888     {
 889         /* If the the rundown wasn't already active, then take the long path */
 890         ExfWaitForRundownProtectionRelease(RunRef);
 891     }
 892 }
 893
 894 /*++
 895  * @name ExRundownCompleted
 896  * INTERNAL MACRO
 897  *
 898  *     The ExRundownCompleted routine completes the rundown of the specified
 899  *     descriptor by setting the active bit.
 900  *
 901  * @param RunRef
 902  *        Pointer to a rundown reference descriptor.
 903  *
 904  * @return None.
 905  *
 906  * @remarks This is the internal macro for system use only.
 907  *
 908  *--*/
 909 FORCEINLINE
 910 VOID
 911 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 912 {
 913     /* Sanity check */
 914     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 915
 916     /* Mark the counter as active */
 917     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 918 }
 919
 920 /* PUSHLOCKS *****************************************************************/
 921
 922 /* FIXME: VERIFY THESE! */
 923
 924 VOID
 925 FASTCALL
 926 ExBlockPushLock(
 927     IN PEX_PUSH_LOCK PushLock,
 928     IN PVOID WaitBlock
 929 );
 930
 931 VOID
 932 FASTCALL
 933 ExfUnblockPushLock(
 934     IN PEX_PUSH_LOCK PushLock,
 935     IN PVOID CurrentWaitBlock
 936 );
 937
 938 VOID
 939 FASTCALL
 940 ExWaitForUnblockPushLock(
 941     IN PEX_PUSH_LOCK PushLock,
 942     IN PVOID WaitBlock
 943 );
 944
 945 /*++
 946  * @name _ExInitializePushLock
 947  * INTERNAL MACRO
 948  *
 949  *     The _ExInitializePushLock macro initializes a PushLock.
 950  *
 951  * @params PushLock
 952  *         Pointer to the pushlock which is to be initialized.
 953  *
 954  * @return None.
 955  *
 956  * @remarks None.
 957  *
 958  *--*/
 959 FORCEINLINE
 960 VOID
 961 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 962 {
 963     /* Set the value to 0 */
 964     PushLock->Ptr = 0;
 965 }
 966 #define ExInitializePushLock _ExInitializePushLock
 967
 968 /*++
 969  * @name ExAcquirePushLockExclusive
 970  * INTERNAL MACRO
 971  *
 972  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 973  *
 974  * @params PushLock
 975  *         Pointer to the pushlock which is to be acquired.
 976  *
 977  * @return None.
 978  *
 979  * @remarks The function attempts the quickest route to acquire the lock, which is
 980  *          to simply set the lock bit.
 981  *          However, if the pushlock is already shared, the slower path is taken.
 982  *
 983  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 984  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 985  *
 986  *--*/
 987 FORCEINLINE
 988 VOID
 989 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 990 {
 991     /* Try acquiring the lock */
 992     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 993     {
 994         /* Someone changed it, use the slow path */
 995         ExfAcquirePushLockExclusive(PushLock);
 996     }
 997
 998     /* Sanity check */
 999     ASSERT(PushLock->Locked);
1000 }
1001
1002 /*++
1003 * @name ExTryToAcquirePushLockExclusive
1004 * INTERNAL MACRO
1005 *
1006 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1007 *
1008 * @params PushLock
1009 *         Pointer to the pushlock which is to be acquired.
1010 *
1011 * @return None.
1012 *
1013 * @remarks The function attempts the quickest route to acquire the lock, which is
1014 *          to simply set the lock bit.
1015 *          However, if the pushlock is already shared, the slower path is taken.
1016 *
1017 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1018 *          This macro should usually be paired up with KeAcquireCriticalRegion.
1019 *
1020 *--*/
1021 FORCEINLINE
1022 BOOLEAN
1023 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1024 {
1025     /* Try acquiring the lock */
1026     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1027     {
1028         /* Can't acquire */
1029         return FALSE;
1030     }
1031
1032     /* Got acquired */
1033     ASSERT (PushLock->Locked);
1034     return TRUE;
1035 }
1036
1037 /*++
1038  * @name ExAcquirePushLockShared
1039  * INTERNAL MACRO
1040  *
1041  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1042  *
1043  * @params PushLock
1044  *         Pointer to the pushlock which is to be acquired.
1045  *
1046  * @return None.
1047  *
1048  * @remarks The function attempts the quickest route to acquire the lock, which is
1049  *          to simply set the lock bit and set the share count to one.
1050  *          However, if the pushlock is already shared, the slower path is taken.
1051  *
1052  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1053  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1054  *
1055  *--*/
1056 FORCEINLINE
1057 VOID
1058 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1059 {
1060     EX_PUSH_LOCK NewValue;
1061
1062     /* Try acquiring the lock */
1063     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1064     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1065     {
1066         /* Someone changed it, use the slow path */
1067         ExfAcquirePushLockShared(PushLock);
1068     }
1069
1070     /* Sanity checks */
1071     ASSERT(PushLock->Locked);
1072 }
1073
1074 /*++
1075  * @name ExConvertPushLockSharedToExclusive
1076  * INTERNAL MACRO
1077  *
1078  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1079  *     pushlock to a shared pushlock.
1080  *
1081  * @params PushLock
1082  *         Pointer to the pushlock which is to be converted.
1083  *
1084  * @return FALSE if conversion failed, TRUE otherwise.
1085  *
1086  * @remarks The function attempts the quickest route to convert the lock, which is
1087  *          to simply set the lock bit and remove any other bits.
1088  *
1089  *--*/
1090 FORCEINLINE
1091 BOOLEAN
1092 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1093 {
1094     EX_PUSH_LOCK OldValue;
1095
1096     /* Set the expected old value */
1097     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1098
1099     /* Try converting the lock */
1100     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1101         OldValue.Ptr)
1102     {
1103         /* Conversion failed */
1104         return FALSE;
1105     }
1106
1107     /* Sanity check */
1108     ASSERT(PushLock->Locked);
1109     return TRUE;
1110 }
1111
1112 /*++
1113  * @name ExWaitOnPushLock
1114  * INTERNAL MACRO
1115  *
1116  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1117  *
1118  * @params PushLock
1119  *         Pointer to a pushlock.
1120  *
1121  * @return None.
1122  *
1123  * @remarks The function attempts to get any exclusive waiters out of their slow
1124  *          path by forcing an instant acquire/release operation.
1125  *
1126  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1127  *
1128  *--*/
1129 FORCEINLINE
1130 VOID
1131 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1132 {
1133     /* Check if we're locked */
1134     if (PushLock->Locked)
1135     {
1136         /* Acquire the lock */
1137         ExfAcquirePushLockExclusive(PushLock);
1138         ASSERT(PushLock->Locked);
1139
1140         /* Release it */
1141         ExfReleasePushLockExclusive(PushLock);
1142     }
1143 }
1144
1145 /*++
1146  * @name ExReleasePushLockShared
1147  * INTERNAL MACRO
1148  *
1149  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1150  *
1151  * @params PushLock
1152  *         Pointer to a previously acquired pushlock.
1153  *
1154  * @return None.
1155  *
1156  * @remarks The function attempts the quickest route to release the lock, which is
1157  *          to simply decrease the share count and remove the lock bit.
1158  *          However, if the pushlock is being waited on then the long path is taken.
1159  *
1160  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1161  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1162  *
1163  *--*/
1164 FORCEINLINE
1165 VOID
1166 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1167 {
1168     EX_PUSH_LOCK OldValue;
1169
1170     /* Sanity checks */
1171     ASSERT(PushLock->Locked);
1172
1173     /* Try to clear the pushlock */
1174     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1175     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1176     {
1177         /* There are still other people waiting on it */
1178         ExfReleasePushLockShared(PushLock);
1179     }
1180 }
1181
1182 /*++
1183  * @name ExReleasePushLockExclusive
1184  * INTERNAL MACRO
1185  *
1186  *     The ExReleasePushLockExclusive macro releases a previously
1187  *     exclusively acquired PushLock.
1188  *
1189  * @params PushLock
1190  *         Pointer to a previously acquired pushlock.
1191  *
1192  * @return None.
1193  *
1194  * @remarks The function attempts the quickest route to release the lock, which is
1195  *          to simply clear the locked bit.
1196  *          However, if the pushlock is being waited on, the slow path is taken
1197  *          in an attempt to wake up the lock.
1198  *
1199  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1200  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1201  *
1202  *--*/
1203 FORCEINLINE
1204 VOID
1205 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1206 {
1207     EX_PUSH_LOCK OldValue;
1208
1209     /* Sanity checks */
1210     ASSERT(PushLock->Locked);
1211
1212     /* Unlock the pushlock */
1213     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1214                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1215
1216     /* Sanity checks */
1217     ASSERT(OldValue.Locked);
1218     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1219
1220     /* Check if anyone is waiting on it and it's not already waking*/
1221     if ((OldValue.Waiting) && !(OldValue.Waking))
1222     {
1223         /* Wake it up */
1224         ExfTryToWakePushLock(PushLock);
1225     }
1226 }
1227
1228 /*++
1229  * @name ExReleasePushLock
1230  * INTERNAL MACRO
1231  *
1232  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1233  *
1234  * @params PushLock
1235  *         Pointer to a previously acquired pushlock.
1236  *
1237  * @return None.
1238  *
1239  * @remarks The function attempts the quickest route to release the lock, which is
1240  *          to simply clear all the fields and decrease the share count if required.
1241  *          However, if the pushlock is being waited on then the long path is taken.
1242  *
1243  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1244  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1245  *
1246  *--*/
1247 FORCEINLINE
1248 VOID
1249 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1250 {
1251     EX_PUSH_LOCK OldValue = *PushLock;
1252     EX_PUSH_LOCK NewValue;
1253
1254     /* Sanity checks */
1255     ASSERT(OldValue.Locked);
1256
1257     /* Check if the pushlock is shared */
1258     if (OldValue.Shared > 1)
1259     {
1260         /* Decrease the share count */
1261         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1262     }
1263     else
1264     {
1265         /* Clear the pushlock entirely */
1266         NewValue.Value = 0;
1267     }
1268
1269     /* Check if nobody is waiting on us and try clearing the lock here */
1270     if ((OldValue.Waiting) ||
1271         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1272          OldValue.Ptr))
1273     {
1274         /* We have waiters, use the long path */
1275         ExfReleasePushLock(PushLock);
1276     }
1277 }
1278
1279 /* FAST MUTEX INLINES *********************************************************/
1280
1281 FORCEINLINE
1282 VOID
1283 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1284 {
1285     PKTHREAD Thread = KeGetCurrentThread();
1286
1287     /* Sanity check */
1288     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1289            (Thread->CombinedApcDisable != 0) ||
1290            (Thread->Teb == NULL) ||
1291            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1292     ASSERT(FastMutex->Owner != Thread);
1293
1294     /* Decrease the count */
1295     if (InterlockedDecrement(&FastMutex->Count))
1296     {
1297         /* Someone is still holding it, use slow path */
1298         KiAcquireFastMutex(FastMutex);
1299     }
1300
1301     /* Set the owner */
1302     FastMutex->Owner = Thread;
1303 }
1304
1305 FORCEINLINE
1306 VOID
1307 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1308 {
1309     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1310            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1311            (KeGetCurrentThread()->Teb == NULL) ||
1312            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1313     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1314
1315     /* Erase the owner */
1316     FastMutex->Owner = NULL;
1317
1318     /* Increase the count */
1319     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1320     {
1321         /* Someone was waiting for it, signal the waiter */
1322         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1323     }
1324 }
1325
1326 FORCEINLINE
1327 VOID
1328 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1329 {
1330     KIRQL OldIrql;
1331     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1332
1333     /* Raise IRQL to APC */
1334     KeRaiseIrql(APC_LEVEL, &OldIrql);
1335
1336     /* Decrease the count */
1337     if (InterlockedDecrement(&FastMutex->Count))
1338     {
1339         /* Someone is still holding it, use slow path */
1340         KiAcquireFastMutex(FastMutex);
1341     }
1342
1343     /* Set the owner and IRQL */
1344     FastMutex->Owner = KeGetCurrentThread();
1345     FastMutex->OldIrql = OldIrql;
1346 }
1347
1348 FORCEINLINE
1349 VOID
1350 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1351 {
1352     KIRQL OldIrql;
1353     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1354
1355     /* Erase the owner */
1356     FastMutex->Owner = NULL;
1357     OldIrql = (KIRQL)FastMutex->OldIrql;
1358
1359     /* Increase the count */
1360     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1361     {
1362         /* Someone was waiting for it, signal the waiter */
1363         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1364     }
1365
1366     /* Lower IRQL back */
1367     KeLowerIrql(OldIrql);
1368 }
1369
1370 FORCEINLINE
1371 BOOLEAN
1372 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1373 {
1374     KIRQL OldIrql;
1375     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1376
1377     /* Raise to APC_LEVEL */
1378     KeRaiseIrql(APC_LEVEL, &OldIrql);
1379
1380     /* Check if we can quickly acquire it */
1381     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1382     {
1383         /* We have, set us as owners */
1384         FastMutex->Owner = KeGetCurrentThread();
1385         FastMutex->OldIrql = OldIrql;
1386         return TRUE;
1387     }
1388     else
1389     {
1390         /* Acquire attempt failed */
1391         KeLowerIrql(OldIrql);
1392         YieldProcessor();
1393         return FALSE;
1394     }
1395 }
1396
1397 FORCEINLINE
1398 VOID
1399 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1400 {
1401     /* Enter the Critical Region */
1402     KeEnterCriticalRegion();
1403
1404     /* Acquire the mutex unsafely */
1405     _ExAcquireFastMutexUnsafe(FastMutex);
1406 }
1407
1408 FORCEINLINE
1409 VOID
1410 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1411 {
1412     /* Release the mutex unsafely */
1413     _ExReleaseFastMutexUnsafe(FastMutex);
1414
1415     /* Leave the critical region */
1416     KeLeaveCriticalRegion();
1417 }
1418
1419 /* OTHER FUNCTIONS **********************************************************/
1420
1421 BOOLEAN
1422 NTAPI
1423 ExTryToAcquireResourceExclusiveLite(
1424     IN PERESOURCE Resource
1425 );
1426
1427 NTSTATUS
1428 ExpSetTimeZoneInformation(
1429     IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1430 );
1431
1432 BOOLEAN
1433 NTAPI
1434 ExAcquireTimeRefreshLock(
1435     IN BOOLEAN Wait
1436 );
1437
1438 VOID
1439 NTAPI
1440 ExReleaseTimeRefreshLock(
1441     VOID
1442 );
1443
1444 VOID
1445 NTAPI
1446 ExUpdateSystemTimeFromCmos(
1447     IN BOOLEAN UpdateInterruptTime,
1448     IN ULONG MaxSepInSeconds
1449 );
1450
1451 VOID
1452 NTAPI
1453 ExAllocateLocallyUniqueId(
1454     OUT LUID *LocallyUniqueId
1455 );
1456
1457 VOID
1458 NTAPI
1459 ExTimerRundown(
1460     VOID
1461 );
1462
1463 VOID
1464 NTAPI
1465 HeadlessInit(
1466     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1467 );
1468
1469 VOID
1470 NTAPI
1471 XIPInit(
1472     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1473 );
1474
1475 #define InterlockedDecrementUL(Addend) \
1476    (ULONG)InterlockedDecrement((PLONG)(Addend))
1477
1478 #define InterlockedIncrementUL(Addend) \
1479    (ULONG)InterlockedIncrement((PLONG)(Addend))
1480
1481 #define InterlockedExchangeUL(Target, Value) \
1482    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1483
1484 #define InterlockedExchangeAddUL(Addend, Value) \
1485    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1486
1487 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1488    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1489
1490 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1491    (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1492
1493 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1494    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))