ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31 extern ULONG ExCriticalWorkerThreads;
  32 extern ULONG ExDelayedWorkerThreads;
  33
  34 extern PVOID ExpDefaultErrorPort;
  35 extern PEPROCESS ExpDefaultErrorPortProcess;
  36
  37 /*
  38  * NT/Cm Version Info variables
  39  */
  40 extern ULONG NtMajorVersion;
  41 extern ULONG NtMinorVersion;
  42 extern ULONG NtBuildNumber;
  43 extern ULONG CmNtSpBuildNumber;
  44 extern ULONG CmNtCSDVersion;
  45 extern ULONG CmNtCSDReleaseType;
  46 extern UNICODE_STRING CmVersionString;
  47 extern UNICODE_STRING CmCSDVersionString;
  48 extern CHAR NtBuildLab[];
  49
  50 // #ifdef _WINKD_
  51 /*
  52  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  53  */
  54 typedef enum _WINKD_WORKER_STATE
  55 {
  56     WinKdWorkerReady = 0,
  57     WinKdWorkerStart,
  58     WinKdWorkerInitialized
  59 } WINKD_WORKER_STATE;
  60
  61 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  62 extern WINKD_WORKER_STATE ExpDebuggerWork;
  63 extern PEPROCESS ExpDebuggerProcessAttach;
  64 extern PEPROCESS ExpDebuggerProcessKill;
  65 extern ULONG_PTR ExpDebuggerPageIn;
  66
  67 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  68 // #endif /* _WINKD_ */
  69
  70 #ifdef _WIN64
  71 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  72 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  73 #else
  74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  76 #endif
  77 #define HANDLE_TAG_BITS (2)
  78 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
  79 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
  80
  81 typedef union _EXHANDLE
  82 {
  83      struct
  84      {
  85          ULONG_PTR TagBits:     HANDLE_TAG_BITS;
  86          ULONG_PTR Index:       HANDLE_INDEX_BITS;
  87          ULONG_PTR KernelFlag : KERNEL_FLAG_BITS;
  88      };
  89      struct
  90      {
  91          ULONG_PTR TagBits2:    HANDLE_TAG_BITS;
  92          ULONG_PTR LowIndex:    HANDLE_LOW_BITS;
  93          ULONG_PTR MidIndex:    HANDLE_HIGH_BITS;
  94          ULONG_PTR HighIndex:   HANDLE_HIGH_BITS;
  95          ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
  96      };
  97      HANDLE GenericHandleOverlay;
  98      ULONG_PTR Value;
  99      ULONG AsULONG;
 100 } EXHANDLE, *PEXHANDLE;
 101
 102 typedef struct _ETIMER
 103 {
 104     KTIMER KeTimer;
 105     KAPC TimerApc;
 106     KDPC TimerDpc;
 107     LIST_ENTRY ActiveTimerListEntry;
 108     KSPIN_LOCK Lock;
 109     LONG Period;
 110     BOOLEAN ApcAssociated;
 111     BOOLEAN WakeTimer;
 112     LIST_ENTRY WakeTimerListEntry;
 113 } ETIMER, *PETIMER;
 114
 115 typedef struct
 116 {
 117     PCALLBACK_OBJECT *CallbackObject;
 118     PWSTR Name;
 119 } SYSTEM_CALLBACKS;
 120
 121 typedef struct _HARDERROR_USER_PARAMETERS
 122 {
 123     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 124     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 125     WCHAR Buffer[ANYSIZE_ARRAY];
 126 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 127
 128 #define MAX_FAST_REFS           7
 129
 130 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 131 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 132 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 133 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 134 #define ExRundownCompleted                              _ExRundownCompleted
 135 #define ExGetPreviousMode                               KeGetPreviousMode
 136
 137
 138 //
 139 // Various bits tagged on the handle or handle table
 140 //
 141 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 142 #define FREE_HANDLE_MASK                -1
 143
 144 //
 145 // Number of entries in each table level
 146 //
 147 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 148 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 149 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 150
 151 //
 152 // Maximum index in each table level before we need another table
 153 //
 154 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 155 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 156 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 157
 158 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 159 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 160 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 161
 162 NTSTATUS
 163 NTAPI
 164 ExGetPoolTagInfo(
 165     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 166     IN ULONG SystemInformationLength,
 167     IN OUT PULONG ReturnLength OPTIONAL
 168 );
 169
 170 typedef struct _UUID_CACHED_VALUES_STRUCT
 171 {
 172     ULONGLONG Time;
 173     LONG AllocatedCount;
 174     UCHAR ClockSeqHiAndReserved;
 175     UCHAR ClockSeqLow;
 176     UCHAR NodeId [ 6 ] ;
 177 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT;
 178
 179 /* INITIALIZATION FUNCTIONS *************************************************/
 180
 181 INIT_FUNCTION
 182 BOOLEAN
 183 NTAPI
 184 ExpWin32kInit(VOID);
 185
 186 VOID
 187 NTAPI
 188 ExInit2(VOID);
 189
 190 VOID
 191 NTAPI
 192 Phase1Initialization(
 193     IN PVOID Context
 194 );
 195
 196 INIT_FUNCTION
 197 VOID
 198 NTAPI
 199 ExpInitializePushLocks(VOID);
 200
 201 BOOLEAN
 202 NTAPI
 203 ExRefreshTimeZoneInformation(
 204     IN PLARGE_INTEGER SystemBootTime
 205 );
 206
 207 INIT_FUNCTION
 208 VOID
 209 NTAPI
 210 ExpInitializeWorkerThreads(VOID);
 211
 212 VOID
 213 NTAPI
 214 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 215
 216 INIT_FUNCTION
 217 VOID
 218 NTAPI
 219 ExpInitLookasideLists(VOID);
 220
 221 INIT_FUNCTION
 222 VOID
 223 NTAPI
 224 ExInitializeSystemLookasideList(
 225     IN PGENERAL_LOOKASIDE List,
 226     IN POOL_TYPE Type,
 227     IN ULONG Size,
 228     IN ULONG Tag,
 229     IN USHORT MaximumDepth,
 230     IN PLIST_ENTRY ListHead
 231 );
 232
 233 INIT_FUNCTION
 234 BOOLEAN
 235 NTAPI
 236 ExpInitializeCallbacks(VOID);
 237
 238 INIT_FUNCTION
 239 VOID
 240 NTAPI
 241 ExpInitUuids(VOID);
 242
 243 INIT_FUNCTION
 244 VOID
 245 NTAPI
 246 ExpInitializeExecutive(
 247     IN ULONG Cpu,
 248     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 249 );
 250
 251 VOID
 252 NTAPI
 253 ExShutdownSystem(VOID);
 254
 255 INIT_FUNCTION
 256 BOOLEAN
 257 NTAPI
 258 ExpInitializeEventImplementation(VOID);
 259
 260 INIT_FUNCTION
 261 BOOLEAN
 262 NTAPI
 263 ExpInitializeKeyedEventImplementation(VOID);
 264
 265 INIT_FUNCTION
 266 BOOLEAN
 267 NTAPI
 268 ExpInitializeEventPairImplementation(VOID);
 269
 270 INIT_FUNCTION
 271 BOOLEAN
 272 NTAPI
 273 ExpInitializeSemaphoreImplementation(VOID);
 274
 275 INIT_FUNCTION
 276 BOOLEAN
 277 NTAPI
 278 ExpInitializeMutantImplementation(VOID);
 279
 280 INIT_FUNCTION
 281 BOOLEAN
 282 NTAPI
 283 ExpInitializeTimerImplementation(VOID);
 284
 285 INIT_FUNCTION
 286 BOOLEAN
 287 NTAPI
 288 ExpInitializeProfileImplementation(VOID);
 289
 290 INIT_FUNCTION
 291 VOID
 292 NTAPI
 293 ExpResourceInitialization(VOID);
 294
 295 INIT_FUNCTION
 296 VOID
 297 NTAPI
 298 ExInitPoolLookasidePointers(VOID);
 299
 300 /* Callback Functions ********************************************************/
 301
 302 VOID
 303 NTAPI
 304 ExInitializeCallBack(
 305     IN OUT PEX_CALLBACK Callback
 306 );
 307
 308 PEX_CALLBACK_ROUTINE_BLOCK
 309 NTAPI
 310 ExAllocateCallBack(
 311     IN PEX_CALLBACK_FUNCTION Function,
 312     IN PVOID Context
 313 );
 314
 315 VOID
 316 NTAPI
 317 ExFreeCallBack(
 318     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 319 );
 320
 321 BOOLEAN
 322 NTAPI
 323 ExCompareExchangeCallBack (
 324     IN OUT PEX_CALLBACK CallBack,
 325     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 326     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 327 );
 328
 329 PEX_CALLBACK_ROUTINE_BLOCK
 330 NTAPI
 331 ExReferenceCallBackBlock(
 332     IN OUT PEX_CALLBACK CallBack
 333 );
 334
 335 VOID
 336 NTAPI
 337 ExDereferenceCallBackBlock(
 338     IN OUT PEX_CALLBACK CallBack,
 339     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 340 );
 341
 342 PEX_CALLBACK_FUNCTION
 343 NTAPI
 344 ExGetCallBackBlockRoutine(
 345     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 346 );
 347
 348 PVOID
 349 NTAPI
 350 ExGetCallBackBlockContext(
 351     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 352 );
 353
 354 VOID
 355 NTAPI
 356 ExWaitForCallBacks(
 357     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 358 );
 359
 360 /* Rundown Functions ********************************************************/
 361
 362 VOID
 363 FASTCALL
 364 ExfInitializeRundownProtection(
 365      OUT PEX_RUNDOWN_REF RunRef
 366 );
 367
 368 VOID
 369 FASTCALL
 370 ExfReInitializeRundownProtection(
 371      OUT PEX_RUNDOWN_REF RunRef
 372 );
 373
 374 BOOLEAN
 375 FASTCALL
 376 ExfAcquireRundownProtection(
 377      IN OUT PEX_RUNDOWN_REF RunRef
 378 );
 379
 380 BOOLEAN
 381 FASTCALL
 382 ExfAcquireRundownProtectionEx(
 383      IN OUT PEX_RUNDOWN_REF RunRef,
 384      IN ULONG Count
 385 );
 386
 387 VOID
 388 FASTCALL
 389 ExfReleaseRundownProtection(
 390      IN OUT PEX_RUNDOWN_REF RunRef
 391 );
 392
 393 VOID
 394 FASTCALL
 395 ExfReleaseRundownProtectionEx(
 396      IN OUT PEX_RUNDOWN_REF RunRef,
 397      IN ULONG Count
 398 );
 399
 400 VOID
 401 FASTCALL
 402 ExfRundownCompleted(
 403      OUT PEX_RUNDOWN_REF RunRef
 404 );
 405
 406 VOID
 407 FASTCALL
 408 ExfWaitForRundownProtectionRelease(
 409      IN OUT PEX_RUNDOWN_REF RunRef
 410 );
 411
 412 /* HANDLE TABLE FUNCTIONS ***************************************************/
 413
 414 typedef BOOLEAN
 415 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 416     PHANDLE_TABLE_ENTRY HandleTableEntry,
 417     HANDLE Handle,
 418     PVOID Context
 419 );
 420
 421 typedef BOOLEAN
 422 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 423     IN PEPROCESS Process,
 424     IN PHANDLE_TABLE HandleTable,
 425     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 426     IN PHANDLE_TABLE_ENTRY NewEntry
 427 );
 428
 429 typedef BOOLEAN
 430 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 431     PHANDLE_TABLE_ENTRY HandleTableEntry,
 432     ULONG_PTR Context
 433 );
 434
 435 INIT_FUNCTION
 436 VOID
 437 NTAPI
 438 ExpInitializeHandleTables(
 439     VOID
 440 );
 441
 442 PHANDLE_TABLE
 443 NTAPI
 444 ExCreateHandleTable(
 445     IN PEPROCESS Process OPTIONAL
 446 );
 447
 448 VOID
 449 NTAPI
 450 ExUnlockHandleTableEntry(
 451     IN PHANDLE_TABLE HandleTable,
 452     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 453 );
 454
 455 HANDLE
 456 NTAPI
 457 ExCreateHandle(
 458     IN PHANDLE_TABLE HandleTable,
 459     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 460 );
 461
 462 VOID
 463 NTAPI
 464 ExDestroyHandleTable(
 465     IN PHANDLE_TABLE HandleTable,
 466     IN PVOID DestroyHandleProcedure OPTIONAL
 467 );
 468
 469 BOOLEAN
 470 NTAPI
 471 ExDestroyHandle(
 472     IN PHANDLE_TABLE HandleTable,
 473     IN HANDLE Handle,
 474     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 475 );
 476
 477 PHANDLE_TABLE_ENTRY
 478 NTAPI
 479 ExMapHandleToPointer(
 480     IN PHANDLE_TABLE HandleTable,
 481     IN HANDLE Handle
 482 );
 483
 484 PHANDLE_TABLE
 485 NTAPI
 486 ExDupHandleTable(
 487     IN PEPROCESS Process,
 488     IN PHANDLE_TABLE HandleTable,
 489     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 490     IN ULONG_PTR Mask
 491 );
 492
 493 BOOLEAN
 494 NTAPI
 495 ExChangeHandle(
 496     IN PHANDLE_TABLE HandleTable,
 497     IN HANDLE Handle,
 498     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 499     IN ULONG_PTR Context
 500 );
 501
 502 VOID
 503 NTAPI
 504 ExSweepHandleTable(
 505     IN PHANDLE_TABLE HandleTable,
 506     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 507     IN PVOID Context
 508 );
 509
 510 PHANDLE_TABLE_ENTRY
 511 NTAPI
 512 ExpLookupHandleTableEntry(
 513     IN PHANDLE_TABLE HandleTable,
 514     IN EXHANDLE Handle
 515 );
 516
 517 BOOLEAN
 518 NTAPI
 519 ExpLockHandleTableEntry(
 520     IN PHANDLE_TABLE HandleTable,
 521     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 522 );
 523
 524 /* PSEH EXCEPTION HANDLING **************************************************/
 525
 526 LONG
 527 NTAPI
 528 ExSystemExceptionFilter(VOID);
 529
 530 /* CALLBACKS *****************************************************************/
 531
 532 FORCEINLINE
 533 VOID
 534 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 535              IN PVOID Context,
 536              IN PVOID Argument1,
 537              IN PVOID Argument2)
 538 {
 539     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 540     PEX_CALLBACK_FUNCTION Function;
 541
 542     /* Reference the block */
 543     CallbackBlock = ExReferenceCallBackBlock(Callback);
 544     if (CallbackBlock)
 545     {
 546         /* Get the function */
 547         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 548
 549         /* Do the callback */
 550         Function(Context, Argument1, Argument2);
 551
 552         /* Now dereference it */
 553         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 554     }
 555 }
 556
 557 /* FAST REFS ******************************************************************/
 558
 559 FORCEINLINE
 560 PVOID
 561 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 562 {
 563     /* Return the unbiased pointer */
 564     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 565 }
 566
 567 FORCEINLINE
 568 ULONG
 569 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 570 {
 571     /* Return the reference count */
 572     return (ULONG)FastRef.RefCnt;
 573 }
 574
 575 FORCEINLINE
 576 VOID
 577 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 578                           IN OPTIONAL PVOID Object)
 579 {
 580     /* Sanity check */
 581     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 582
 583     /* Check if an object is being set */
 584     if (!Object)
 585     {
 586         /* Clear the field */
 587         FastRef->Object = NULL;
 588     }
 589     else
 590     {
 591         /* Otherwise, we assume the object was referenced and is ready */
 592         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 593     }
 594 }
 595
 596 FORCEINLINE
 597 EX_FAST_REF
 598 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 599 {
 600     EX_FAST_REF OldValue, NewValue;
 601
 602     /* Start reference loop */
 603     for (;;)
 604     {
 605         /* Get the current reference count */
 606         OldValue = *FastRef;
 607         if (OldValue.RefCnt)
 608         {
 609             /* Increase the reference count */
 610             NewValue.Value = OldValue.Value - 1;
 611             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 612                                                 NewValue.Object,
 613                                                 OldValue.Object);
 614             if (NewValue.Object != OldValue.Object) continue;
 615         }
 616
 617         /* We are done */
 618         break;
 619     }
 620
 621     /* Return the old value */
 622     return OldValue;
 623 }
 624
 625 FORCEINLINE
 626 BOOLEAN
 627 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 628                       IN PVOID Object)
 629 {
 630     EX_FAST_REF OldValue, NewValue;
 631
 632     /* Sanity checks */
 633     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 634
 635     /* Start update loop */
 636     for (;;)
 637     {
 638         /* Get the current reference count */
 639         OldValue = *FastRef;
 640
 641         /* Check if the current count is too high or if the pointer changed */
 642         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 643             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 644         {
 645             /* Fail */
 646             return FALSE;
 647         }
 648
 649         /* Update the reference count */
 650         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 651         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 652                                             NewValue.Object,
 653                                             OldValue.Object);
 654         if (NewValue.Object != OldValue.Object) continue;
 655
 656         /* We are done */
 657         break;
 658     }
 659
 660     /* Return success */
 661     return TRUE;
 662 }
 663
 664 FORCEINLINE
 665 BOOLEAN
 666 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 667                        IN PVOID Object)
 668 {
 669     EX_FAST_REF OldValue, NewValue;
 670
 671     /* Sanity checks */
 672     ASSERT(Object != NULL);
 673     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 674
 675     /* Start reference loop */
 676     for (;;)
 677     {
 678         /* Get the current reference count */
 679         OldValue = *FastRef;
 680
 681         /* Check if we're full if if the pointer changed */
 682         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 683
 684         /* Decrease the reference count */
 685         NewValue.Value = OldValue.Value + 1;
 686         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 687                                             NewValue.Object,
 688                                             OldValue.Object);
 689         if (NewValue.Object != OldValue.Object) continue;
 690
 691         /* We are done */
 692         break;
 693     }
 694
 695     /* Return success */
 696     return TRUE;
 697 }
 698
 699 FORCEINLINE
 700 EX_FAST_REF
 701 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 702                     IN PVOID Object)
 703 {
 704     EX_FAST_REF NewValue, OldValue;
 705
 706     /* Sanity check */
 707     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 708
 709     /* Check if an object is being set */
 710     if (!Object)
 711     {
 712         /* Clear the field */
 713         NewValue.Object = NULL;
 714     }
 715     else
 716     {
 717         /* Otherwise, we assume the object was referenced and is ready */
 718         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 719     }
 720
 721     /* Update the object */
 722     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 723     return OldValue;
 724 }
 725
 726 FORCEINLINE
 727 EX_FAST_REF
 728 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 729                            IN PVOID Object,
 730                            IN PVOID OldObject)
 731 {
 732     EX_FAST_REF OldValue, NewValue;
 733
 734     /* Sanity check and start swap loop */
 735     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 736     for (;;)
 737     {
 738         /* Get the current value */
 739         OldValue = *FastRef;
 740
 741         /* Make sure there's enough references to swap */
 742         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 743
 744         /* Check if we have an object to swap */
 745         if (Object)
 746         {
 747             /* Set up the value with maximum fast references */
 748             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 749         }
 750         else
 751         {
 752             /* Write the object address itself (which is empty) */
 753             NewValue.Value = (ULONG_PTR)Object;
 754         }
 755
 756         /* Do the actual compare exchange */
 757         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 758                                             NewValue.Object,
 759                                             OldValue.Object);
 760         if (NewValue.Object != OldValue.Object) continue;
 761
 762         /* All done */
 763         break;
 764     }
 765
 766     /* Return the old value */
 767     return OldValue;
 768 }
 769
 770 /* RUNDOWN *******************************************************************/
 771
 772 FORCEINLINE
 773 PEX_RUNDOWN_REF
 774 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 775                              IN ULONG ProcNumber)
 776 {
 777     return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 778                              RunRefCacheAware->RunRefSize *
 779                              (ProcNumber % RunRefCacheAware->Number));
 780 }
 781
 782 /*++
 783  * @name ExfAcquireRundownProtection
 784  * INTERNAL MACRO
 785  *
 786  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 787  *     the specified descriptor.
 788  *
 789  * @param RunRef
 790  *        Pointer to a rundown reference descriptor.
 791  *
 792  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 793  *
 794  * @remarks This is the internal macro for system use only.In case the rundown
 795  *          was active, then the slow-path will be called through the exported
 796  *          function.
 797  *
 798  *--*/
 799 FORCEINLINE
 800 BOOLEAN
 801 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 802 {
 803     ULONG_PTR Value, NewValue;
 804
 805     /* Get the current value and mask the active bit */
 806     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 807
 808     /* Add a reference */
 809     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 810
 811     /* Change the value */
 812     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 813     if (NewValue != Value)
 814     {
 815         /* Rundown was active, use long path */
 816         return ExfAcquireRundownProtection(RunRef);
 817     }
 818
 819     /* Success */
 820     return TRUE;
 821 }
 822
 823 /*++
 824  * @name ExReleaseRundownProtection
 825  * INTERNAL MACRO
 826  *
 827  *     The ExReleaseRundownProtection routine releases rundown protection for
 828  *     the specified descriptor.
 829  *
 830  * @param RunRef
 831  *        Pointer to a rundown reference descriptor.
 832  *
 833  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 834  *
 835  * @remarks This is the internal macro for system use only.In case the rundown
 836  *          was active, then the slow-path will be called through the exported
 837  *          function.
 838  *
 839  *--*/
 840 FORCEINLINE
 841 VOID
 842 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 843 {
 844     ULONG_PTR Value, NewValue;
 845
 846     /* Get the current value and mask the active bit */
 847     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 848
 849     /* Remove a reference */
 850     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 851
 852     /* Change the value */
 853     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 854
 855     /* Check if the rundown was active */
 856     if (NewValue != Value)
 857     {
 858         /* Rundown was active, use long path */
 859         ExfReleaseRundownProtection(RunRef);
 860     }
 861     else
 862     {
 863         /* Sanity check */
 864         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 865     }
 866 }
 867
 868 /*++
 869  * @name ExInitializeRundownProtection
 870  * INTERNAL MACRO
 871  *
 872  *     The ExInitializeRundownProtection routine initializes a rundown
 873  *     protection descriptor.
 874  *
 875  * @param RunRef
 876  *        Pointer to a rundown reference descriptor.
 877  *
 878  * @return None.
 879  *
 880  * @remarks This is the internal macro for system use only.
 881  *
 882  *--*/
 883 FORCEINLINE
 884 VOID
 885 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 886 {
 887     /* Set the count to zero */
 888     RunRef->Count = 0;
 889 }
 890
 891 /*++
 892  * @name ExWaitForRundownProtectionRelease
 893  * INTERNAL MACRO
 894  *
 895  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 896  *     rundown descriptor has been released.
 897  *
 898  * @param RunRef
 899  *        Pointer to a rundown reference descriptor.
 900  *
 901  * @return None.
 902  *
 903  * @remarks This is the internal macro for system use only. If a wait is actually
 904  *          necessary, then the slow path is taken through the exported function.
 905  *
 906  *--*/
 907 FORCEINLINE
 908 VOID
 909 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 910 {
 911     ULONG_PTR Value;
 912
 913     /* Set the active bit */
 914     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 915     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 916     {
 917         /* If the the rundown wasn't already active, then take the long path */
 918         ExfWaitForRundownProtectionRelease(RunRef);
 919     }
 920 }
 921
 922 /*++
 923  * @name ExRundownCompleted
 924  * INTERNAL MACRO
 925  *
 926  *     The ExRundownCompleted routine completes the rundown of the specified
 927  *     descriptor by setting the active bit.
 928  *
 929  * @param RunRef
 930  *        Pointer to a rundown reference descriptor.
 931  *
 932  * @return None.
 933  *
 934  * @remarks This is the internal macro for system use only.
 935  *
 936  *--*/
 937 FORCEINLINE
 938 VOID
 939 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 940 {
 941     /* Sanity check */
 942     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 943
 944     /* Mark the counter as active */
 945     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 946 }
 947
 948 /* PUSHLOCKS *****************************************************************/
 949
 950 /* FIXME: VERIFY THESE! */
 951
 952 VOID
 953 FASTCALL
 954 ExBlockPushLock(
 955     IN PEX_PUSH_LOCK PushLock,
 956     IN PVOID WaitBlock
 957 );
 958
 959 VOID
 960 FASTCALL
 961 ExfUnblockPushLock(
 962     IN PEX_PUSH_LOCK PushLock,
 963     IN PVOID CurrentWaitBlock
 964 );
 965
 966 VOID
 967 FASTCALL
 968 ExWaitForUnblockPushLock(
 969     IN PEX_PUSH_LOCK PushLock,
 970     IN PVOID WaitBlock
 971 );
 972
 973 /*++
 974  * @name _ExInitializePushLock
 975  * INTERNAL MACRO
 976  *
 977  *     The _ExInitializePushLock macro initializes a PushLock.
 978  *
 979  * @params PushLock
 980  *         Pointer to the pushlock which is to be initialized.
 981  *
 982  * @return None.
 983  *
 984  * @remarks None.
 985  *
 986  *--*/
 987 FORCEINLINE
 988 VOID
 989 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 990 {
 991     /* Set the value to 0 */
 992     PushLock->Ptr = 0;
 993 }
 994 #define ExInitializePushLock _ExInitializePushLock
 995
 996 /*++
 997  * @name ExAcquirePushLockExclusive
 998  * INTERNAL MACRO
 999  *
1000  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1001  *
1002  * @params PushLock
1003  *         Pointer to the pushlock which is to be acquired.
1004  *
1005  * @return None.
1006  *
1007  * @remarks The function attempts the quickest route to acquire the lock, which is
1008  *          to simply set the lock bit.
1009  *          However, if the pushlock is already shared, the slower path is taken.
1010  *
1011  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1012  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1013  *
1014  *--*/
1015 FORCEINLINE
1016 VOID
1017 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1018 {
1019     /* Try acquiring the lock */
1020     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1021     {
1022         /* Someone changed it, use the slow path */
1023         ExfAcquirePushLockExclusive(PushLock);
1024     }
1025
1026     /* Sanity check */
1027     ASSERT(PushLock->Locked);
1028 }
1029
1030 /*++
1031 * @name ExTryToAcquirePushLockExclusive
1032 * INTERNAL MACRO
1033 *
1034 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1035 *
1036 * @params PushLock
1037 *         Pointer to the pushlock which is to be acquired.
1038 *
1039 * @return None.
1040 *
1041 * @remarks The function attempts the quickest route to acquire the lock, which is
1042 *          to simply set the lock bit.
1043 *          However, if the pushlock is already shared, the slower path is taken.
1044 *
1045 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1046 *          This macro should usually be paired up with KeAcquireCriticalRegion.
1047 *
1048 *--*/
1049 FORCEINLINE
1050 BOOLEAN
1051 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1052 {
1053     /* Try acquiring the lock */
1054     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1055     {
1056         /* Can't acquire */
1057         return FALSE;
1058     }
1059
1060     /* Got acquired */
1061     ASSERT (PushLock->Locked);
1062     return TRUE;
1063 }
1064
1065 /*++
1066  * @name ExAcquirePushLockShared
1067  * INTERNAL MACRO
1068  *
1069  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1070  *
1071  * @params PushLock
1072  *         Pointer to the pushlock which is to be acquired.
1073  *
1074  * @return None.
1075  *
1076  * @remarks The function attempts the quickest route to acquire the lock, which is
1077  *          to simply set the lock bit and set the share count to one.
1078  *          However, if the pushlock is already shared, the slower path is taken.
1079  *
1080  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1081  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1082  *
1083  *--*/
1084 FORCEINLINE
1085 VOID
1086 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1087 {
1088     EX_PUSH_LOCK NewValue;
1089
1090     /* Try acquiring the lock */
1091     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1092     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1093     {
1094         /* Someone changed it, use the slow path */
1095         ExfAcquirePushLockShared(PushLock);
1096     }
1097
1098     /* Sanity checks */
1099     ASSERT(PushLock->Locked);
1100 }
1101
1102 /*++
1103  * @name ExConvertPushLockSharedToExclusive
1104  * INTERNAL MACRO
1105  *
1106  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1107  *     pushlock to a shared pushlock.
1108  *
1109  * @params PushLock
1110  *         Pointer to the pushlock which is to be converted.
1111  *
1112  * @return FALSE if conversion failed, TRUE otherwise.
1113  *
1114  * @remarks The function attempts the quickest route to convert the lock, which is
1115  *          to simply set the lock bit and remove any other bits.
1116  *
1117  *--*/
1118 FORCEINLINE
1119 BOOLEAN
1120 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1121 {
1122     EX_PUSH_LOCK OldValue;
1123
1124     /* Set the expected old value */
1125     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1126
1127     /* Try converting the lock */
1128     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1129         OldValue.Ptr)
1130     {
1131         /* Conversion failed */
1132         return FALSE;
1133     }
1134
1135     /* Sanity check */
1136     ASSERT(PushLock->Locked);
1137     return TRUE;
1138 }
1139
1140 /*++
1141  * @name ExWaitOnPushLock
1142  * INTERNAL MACRO
1143  *
1144  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1145  *
1146  * @params PushLock
1147  *         Pointer to a pushlock.
1148  *
1149  * @return None.
1150  *
1151  * @remarks The function attempts to get any exclusive waiters out of their slow
1152  *          path by forcing an instant acquire/release operation.
1153  *
1154  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1155  *
1156  *--*/
1157 FORCEINLINE
1158 VOID
1159 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1160 {
1161     /* Check if we're locked */
1162     if (PushLock->Locked)
1163     {
1164         /* Acquire the lock */
1165         ExfAcquirePushLockExclusive(PushLock);
1166         ASSERT(PushLock->Locked);
1167
1168         /* Release it */
1169         ExfReleasePushLockExclusive(PushLock);
1170     }
1171 }
1172
1173 /*++
1174  * @name ExReleasePushLockShared
1175  * INTERNAL MACRO
1176  *
1177  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1178  *
1179  * @params PushLock
1180  *         Pointer to a previously acquired pushlock.
1181  *
1182  * @return None.
1183  *
1184  * @remarks The function attempts the quickest route to release the lock, which is
1185  *          to simply decrease the share count and remove the lock bit.
1186  *          However, if the pushlock is being waited on then the long path is taken.
1187  *
1188  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1189  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1190  *
1191  *--*/
1192 FORCEINLINE
1193 VOID
1194 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1195 {
1196     EX_PUSH_LOCK OldValue;
1197
1198     /* Sanity checks */
1199     ASSERT(PushLock->Locked);
1200
1201     /* Try to clear the pushlock */
1202     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1203     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1204     {
1205         /* There are still other people waiting on it */
1206         ExfReleasePushLockShared(PushLock);
1207     }
1208 }
1209
1210 /*++
1211  * @name ExReleasePushLockExclusive
1212  * INTERNAL MACRO
1213  *
1214  *     The ExReleasePushLockExclusive macro releases a previously
1215  *     exclusively acquired PushLock.
1216  *
1217  * @params PushLock
1218  *         Pointer to a previously acquired pushlock.
1219  *
1220  * @return None.
1221  *
1222  * @remarks The function attempts the quickest route to release the lock, which is
1223  *          to simply clear the locked bit.
1224  *          However, if the pushlock is being waited on, the slow path is taken
1225  *          in an attempt to wake up the lock.
1226  *
1227  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1228  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1229  *
1230  *--*/
1231 FORCEINLINE
1232 VOID
1233 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1234 {
1235     EX_PUSH_LOCK OldValue;
1236
1237     /* Sanity checks */
1238     ASSERT(PushLock->Locked);
1239
1240     /* Unlock the pushlock */
1241     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1242                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1243
1244     /* Sanity checks */
1245     ASSERT(OldValue.Locked);
1246     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1247
1248     /* Check if anyone is waiting on it and it's not already waking*/
1249     if ((OldValue.Waiting) && !(OldValue.Waking))
1250     {
1251         /* Wake it up */
1252         ExfTryToWakePushLock(PushLock);
1253     }
1254 }
1255
1256 /*++
1257  * @name ExReleasePushLock
1258  * INTERNAL MACRO
1259  *
1260  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1261  *
1262  * @params PushLock
1263  *         Pointer to a previously acquired pushlock.
1264  *
1265  * @return None.
1266  *
1267  * @remarks The function attempts the quickest route to release the lock, which is
1268  *          to simply clear all the fields and decrease the share count if required.
1269  *          However, if the pushlock is being waited on then the long path is taken.
1270  *
1271  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1272  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1273  *
1274  *--*/
1275 FORCEINLINE
1276 VOID
1277 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1278 {
1279     EX_PUSH_LOCK OldValue = *PushLock;
1280     EX_PUSH_LOCK NewValue;
1281
1282     /* Sanity checks */
1283     ASSERT(OldValue.Locked);
1284
1285     /* Check if the pushlock is shared */
1286     if (OldValue.Shared > 1)
1287     {
1288         /* Decrease the share count */
1289         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1290     }
1291     else
1292     {
1293         /* Clear the pushlock entirely */
1294         NewValue.Value = 0;
1295     }
1296
1297     /* Check if nobody is waiting on us and try clearing the lock here */
1298     if ((OldValue.Waiting) ||
1299         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1300          OldValue.Ptr))
1301     {
1302         /* We have waiters, use the long path */
1303         ExfReleasePushLock(PushLock);
1304     }
1305 }
1306
1307 /* FAST MUTEX INLINES *********************************************************/
1308
1309 FORCEINLINE
1310 VOID
1311 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1312 {
1313     PKTHREAD Thread = KeGetCurrentThread();
1314
1315     /* Sanity check */
1316     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1317            (Thread->CombinedApcDisable != 0) ||
1318            (Thread->Teb == NULL) ||
1319            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1320     ASSERT(FastMutex->Owner != Thread);
1321
1322     /* Decrease the count */
1323     if (InterlockedDecrement(&FastMutex->Count))
1324     {
1325         /* Someone is still holding it, use slow path */
1326         KiAcquireFastMutex(FastMutex);
1327     }
1328
1329     /* Set the owner */
1330     FastMutex->Owner = Thread;
1331 }
1332
1333 FORCEINLINE
1334 VOID
1335 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1336 {
1337     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1338            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1339            (KeGetCurrentThread()->Teb == NULL) ||
1340            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1341     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1342
1343     /* Erase the owner */
1344     FastMutex->Owner = NULL;
1345
1346     /* Increase the count */
1347     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1348     {
1349         /* Someone was waiting for it, signal the waiter */
1350         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1351     }
1352 }
1353
1354 FORCEINLINE
1355 VOID
1356 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1357 {
1358     KIRQL OldIrql;
1359     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1360
1361     /* Raise IRQL to APC */
1362     KeRaiseIrql(APC_LEVEL, &OldIrql);
1363
1364     /* Decrease the count */
1365     if (InterlockedDecrement(&FastMutex->Count))
1366     {
1367         /* Someone is still holding it, use slow path */
1368         KiAcquireFastMutex(FastMutex);
1369     }
1370
1371     /* Set the owner and IRQL */
1372     FastMutex->Owner = KeGetCurrentThread();
1373     FastMutex->OldIrql = OldIrql;
1374 }
1375
1376 FORCEINLINE
1377 VOID
1378 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1379 {
1380     KIRQL OldIrql;
1381     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1382
1383     /* Erase the owner */
1384     FastMutex->Owner = NULL;
1385     OldIrql = (KIRQL)FastMutex->OldIrql;
1386
1387     /* Increase the count */
1388     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1389     {
1390         /* Someone was waiting for it, signal the waiter */
1391         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1392     }
1393
1394     /* Lower IRQL back */
1395     KeLowerIrql(OldIrql);
1396 }
1397
1398 FORCEINLINE
1399 BOOLEAN
1400 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1401 {
1402     KIRQL OldIrql;
1403     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1404
1405     /* Raise to APC_LEVEL */
1406     KeRaiseIrql(APC_LEVEL, &OldIrql);
1407
1408     /* Check if we can quickly acquire it */
1409     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1410     {
1411         /* We have, set us as owners */
1412         FastMutex->Owner = KeGetCurrentThread();
1413         FastMutex->OldIrql = OldIrql;
1414         return TRUE;
1415     }
1416     else
1417     {
1418         /* Acquire attempt failed */
1419         KeLowerIrql(OldIrql);
1420         YieldProcessor();
1421         return FALSE;
1422     }
1423 }
1424
1425 FORCEINLINE
1426 VOID
1427 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1428 {
1429     /* Enter the Critical Region */
1430     KeEnterCriticalRegion();
1431
1432     /* Acquire the mutex unsafely */
1433     _ExAcquireFastMutexUnsafe(FastMutex);
1434 }
1435
1436 FORCEINLINE
1437 VOID
1438 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1439 {
1440     /* Release the mutex unsafely */
1441     _ExReleaseFastMutexUnsafe(FastMutex);
1442
1443     /* Leave the critical region */
1444     KeLeaveCriticalRegion();
1445 }
1446
1447 /* OTHER FUNCTIONS **********************************************************/
1448
1449 BOOLEAN
1450 NTAPI
1451 ExTryToAcquireResourceExclusiveLite(
1452     IN PERESOURCE Resource
1453 );
1454
1455 NTSTATUS
1456 ExpSetTimeZoneInformation(
1457     IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1458 );
1459
1460 BOOLEAN
1461 NTAPI
1462 ExAcquireTimeRefreshLock(
1463     IN BOOLEAN Wait
1464 );
1465
1466 VOID
1467 NTAPI
1468 ExReleaseTimeRefreshLock(
1469     VOID
1470 );
1471
1472 VOID
1473 NTAPI
1474 ExUpdateSystemTimeFromCmos(
1475     IN BOOLEAN UpdateInterruptTime,
1476     IN ULONG MaxSepInSeconds
1477 );
1478
1479 VOID
1480 NTAPI
1481 ExAllocateLocallyUniqueId(
1482     OUT LUID *LocallyUniqueId
1483 );
1484
1485 VOID
1486 NTAPI
1487 ExTimerRundown(
1488     VOID
1489 );
1490
1491 INIT_FUNCTION
1492 VOID
1493 NTAPI
1494 HeadlessInit(
1495     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1496 );
1497
1498 INIT_FUNCTION
1499 VOID
1500 NTAPI
1501 XIPInit(
1502     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1503 );
1504
1505 #define InterlockedDecrementUL(Addend) \
1506    (ULONG)InterlockedDecrement((PLONG)(Addend))
1507
1508 #define InterlockedIncrementUL(Addend) \
1509    (ULONG)InterlockedIncrement((PLONG)(Addend))
1510
1511 #define InterlockedExchangeUL(Target, Value) \
1512    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1513
1514 #define InterlockedExchangeAddUL(Addend, Value) \
1515    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1516
1517 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1518    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1519
1520 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1521    (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1522
1523 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1524    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))