ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31 extern ULONG ExCriticalWorkerThreads;
  32 extern ULONG ExDelayedWorkerThreads;
  33
  34 extern PVOID ExpDefaultErrorPort;
  35 extern PEPROCESS ExpDefaultErrorPortProcess;
  36
  37 /*
  38  * NT/Cm Version Info variables
  39  */
  40 extern ULONG NtMajorVersion;
  41 extern ULONG NtMinorVersion;
  42 extern ULONG NtBuildNumber;
  43 extern ULONG CmNtSpBuildNumber;
  44 extern ULONG CmNtCSDVersion;
  45 extern ULONG CmNtCSDReleaseType;
  46 extern UNICODE_STRING CmVersionString;
  47 extern UNICODE_STRING CmCSDVersionString;
  48 extern CHAR NtBuildLab[];
  49
  50 // #ifdef _WINKD_
  51 /*
  52  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  53  */
  54 typedef enum _WINKD_WORKER_STATE
  55 {
  56     WinKdWorkerReady = 0,
  57     WinKdWorkerStart,
  58     WinKdWorkerInitialized
  59 } WINKD_WORKER_STATE;
  60
  61 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  62 extern WINKD_WORKER_STATE ExpDebuggerWork;
  63 extern PEPROCESS ExpDebuggerProcessAttach;
  64 extern PEPROCESS ExpDebuggerProcessKill;
  65 extern ULONG_PTR ExpDebuggerPageIn;
  66
  67 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  68 // #endif /* _WINKD_ */
  69
  70 #ifdef _WIN64
  71 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  72 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  73 #else
  74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  76 #endif
  77 #define HANDLE_TAG_BITS (2)
  78 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
  79 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
  80
  81 typedef union _EXHANDLE
  82 {
  83      struct
  84      {
  85          ULONG_PTR TagBits:     HANDLE_TAG_BITS;
  86          ULONG_PTR Index:       HANDLE_INDEX_BITS;
  87          ULONG_PTR KernelFlag : KERNEL_FLAG_BITS;
  88      };
  89      struct
  90      {
  91          ULONG_PTR TagBits2:    HANDLE_TAG_BITS;
  92          ULONG_PTR LowIndex:    HANDLE_LOW_BITS;
  93          ULONG_PTR MidIndex:    HANDLE_HIGH_BITS;
  94          ULONG_PTR HighIndex:   HANDLE_HIGH_BITS;
  95          ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
  96      };
  97      HANDLE GenericHandleOverlay;
  98      ULONG_PTR Value;
  99      ULONG AsULONG;
 100 } EXHANDLE, *PEXHANDLE;
 101
 102 typedef struct _ETIMER
 103 {
 104     KTIMER KeTimer;
 105     KAPC TimerApc;
 106     KDPC TimerDpc;
 107     LIST_ENTRY ActiveTimerListEntry;
 108     KSPIN_LOCK Lock;
 109     LONG Period;
 110     BOOLEAN ApcAssociated;
 111     BOOLEAN WakeTimer;
 112     LIST_ENTRY WakeTimerListEntry;
 113 } ETIMER, *PETIMER;
 114
 115 typedef struct
 116 {
 117     PCALLBACK_OBJECT *CallbackObject;
 118     PWSTR Name;
 119 } SYSTEM_CALLBACKS;
 120
 121 typedef struct _HARDERROR_USER_PARAMETERS
 122 {
 123     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 124     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 125     WCHAR Buffer[ANYSIZE_ARRAY];
 126 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 127
 128 #define MAX_FAST_REFS           7
 129
 130 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 131 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 132 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 133 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 134 #define ExRundownCompleted                              _ExRundownCompleted
 135 #define ExGetPreviousMode                               KeGetPreviousMode
 136
 137
 138 //
 139 // Various bits tagged on the handle or handle table
 140 //
 141 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 142 #define FREE_HANDLE_MASK                -1
 143
 144 //
 145 // Number of entries in each table level
 146 //
 147 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 148 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 149 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 150
 151 //
 152 // Maximum index in each table level before we need another table
 153 //
 154 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 155 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 156 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 157
 158 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 159 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 160 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 161
 162 NTSTATUS
 163 NTAPI
 164 ExGetPoolTagInfo(
 165     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 166     IN ULONG SystemInformationLength,
 167     IN OUT PULONG ReturnLength OPTIONAL
 168 );
 169
 170 /* INITIALIZATION FUNCTIONS *************************************************/
 171
 172 INIT_FUNCTION
 173 BOOLEAN
 174 NTAPI
 175 ExpWin32kInit(VOID);
 176
 177 VOID
 178 NTAPI
 179 ExInit2(VOID);
 180
 181 VOID
 182 NTAPI
 183 Phase1Initialization(
 184     IN PVOID Context
 185 );
 186
 187 INIT_FUNCTION
 188 VOID
 189 NTAPI
 190 ExpInitializePushLocks(VOID);
 191
 192 BOOLEAN
 193 NTAPI
 194 ExRefreshTimeZoneInformation(
 195     IN PLARGE_INTEGER SystemBootTime
 196 );
 197
 198 INIT_FUNCTION
 199 VOID
 200 NTAPI
 201 ExpInitializeWorkerThreads(VOID);
 202
 203 VOID
 204 NTAPI
 205 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 206
 207 INIT_FUNCTION
 208 VOID
 209 NTAPI
 210 ExpInitLookasideLists(VOID);
 211
 212 INIT_FUNCTION
 213 VOID
 214 NTAPI
 215 ExInitializeSystemLookasideList(
 216     IN PGENERAL_LOOKASIDE List,
 217     IN POOL_TYPE Type,
 218     IN ULONG Size,
 219     IN ULONG Tag,
 220     IN USHORT MaximumDepth,
 221     IN PLIST_ENTRY ListHead
 222 );
 223
 224 INIT_FUNCTION
 225 BOOLEAN
 226 NTAPI
 227 ExpInitializeCallbacks(VOID);
 228
 229 INIT_FUNCTION
 230 VOID
 231 NTAPI
 232 ExpInitUuids(VOID);
 233
 234 INIT_FUNCTION
 235 VOID
 236 NTAPI
 237 ExpInitializeExecutive(
 238     IN ULONG Cpu,
 239     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 240 );
 241
 242 VOID
 243 NTAPI
 244 ExShutdownSystem(VOID);
 245
 246 INIT_FUNCTION
 247 BOOLEAN
 248 NTAPI
 249 ExpInitializeEventImplementation(VOID);
 250
 251 INIT_FUNCTION
 252 BOOLEAN
 253 NTAPI
 254 ExpInitializeKeyedEventImplementation(VOID);
 255
 256 INIT_FUNCTION
 257 BOOLEAN
 258 NTAPI
 259 ExpInitializeEventPairImplementation(VOID);
 260
 261 INIT_FUNCTION
 262 BOOLEAN
 263 NTAPI
 264 ExpInitializeSemaphoreImplementation(VOID);
 265
 266 INIT_FUNCTION
 267 BOOLEAN
 268 NTAPI
 269 ExpInitializeMutantImplementation(VOID);
 270
 271 INIT_FUNCTION
 272 BOOLEAN
 273 NTAPI
 274 ExpInitializeTimerImplementation(VOID);
 275
 276 INIT_FUNCTION
 277 BOOLEAN
 278 NTAPI
 279 ExpInitializeProfileImplementation(VOID);
 280
 281 INIT_FUNCTION
 282 VOID
 283 NTAPI
 284 ExpResourceInitialization(VOID);
 285
 286 INIT_FUNCTION
 287 VOID
 288 NTAPI
 289 ExInitPoolLookasidePointers(VOID);
 290
 291 /* Callback Functions ********************************************************/
 292
 293 VOID
 294 NTAPI
 295 ExInitializeCallBack(
 296     IN OUT PEX_CALLBACK Callback
 297 );
 298
 299 PEX_CALLBACK_ROUTINE_BLOCK
 300 NTAPI
 301 ExAllocateCallBack(
 302     IN PEX_CALLBACK_FUNCTION Function,
 303     IN PVOID Context
 304 );
 305
 306 VOID
 307 NTAPI
 308 ExFreeCallBack(
 309     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 310 );
 311
 312 BOOLEAN
 313 NTAPI
 314 ExCompareExchangeCallBack (
 315     IN OUT PEX_CALLBACK CallBack,
 316     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 317     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 318 );
 319
 320 PEX_CALLBACK_ROUTINE_BLOCK
 321 NTAPI
 322 ExReferenceCallBackBlock(
 323     IN OUT PEX_CALLBACK CallBack
 324 );
 325
 326 VOID
 327 NTAPI
 328 ExDereferenceCallBackBlock(
 329     IN OUT PEX_CALLBACK CallBack,
 330     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 331 );
 332
 333 PEX_CALLBACK_FUNCTION
 334 NTAPI
 335 ExGetCallBackBlockRoutine(
 336     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 337 );
 338
 339 PVOID
 340 NTAPI
 341 ExGetCallBackBlockContext(
 342     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 343 );
 344
 345 VOID
 346 NTAPI
 347 ExWaitForCallBacks(
 348     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 349 );
 350
 351 /* Rundown Functions ********************************************************/
 352
 353 VOID
 354 FASTCALL
 355 ExfInitializeRundownProtection(
 356      OUT PEX_RUNDOWN_REF RunRef
 357 );
 358
 359 VOID
 360 FASTCALL
 361 ExfReInitializeRundownProtection(
 362      OUT PEX_RUNDOWN_REF RunRef
 363 );
 364
 365 BOOLEAN
 366 FASTCALL
 367 ExfAcquireRundownProtection(
 368      IN OUT PEX_RUNDOWN_REF RunRef
 369 );
 370
 371 BOOLEAN
 372 FASTCALL
 373 ExfAcquireRundownProtectionEx(
 374      IN OUT PEX_RUNDOWN_REF RunRef,
 375      IN ULONG Count
 376 );
 377
 378 VOID
 379 FASTCALL
 380 ExfReleaseRundownProtection(
 381      IN OUT PEX_RUNDOWN_REF RunRef
 382 );
 383
 384 VOID
 385 FASTCALL
 386 ExfReleaseRundownProtectionEx(
 387      IN OUT PEX_RUNDOWN_REF RunRef,
 388      IN ULONG Count
 389 );
 390
 391 VOID
 392 FASTCALL
 393 ExfRundownCompleted(
 394      OUT PEX_RUNDOWN_REF RunRef
 395 );
 396
 397 VOID
 398 FASTCALL
 399 ExfWaitForRundownProtectionRelease(
 400      IN OUT PEX_RUNDOWN_REF RunRef
 401 );
 402
 403 /* HANDLE TABLE FUNCTIONS ***************************************************/
 404
 405 typedef BOOLEAN
 406 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 407     PHANDLE_TABLE_ENTRY HandleTableEntry,
 408     HANDLE Handle,
 409     PVOID Context
 410 );
 411
 412 typedef BOOLEAN
 413 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 414     IN PEPROCESS Process,
 415     IN PHANDLE_TABLE HandleTable,
 416     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 417     IN PHANDLE_TABLE_ENTRY NewEntry
 418 );
 419
 420 typedef BOOLEAN
 421 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 422     PHANDLE_TABLE_ENTRY HandleTableEntry,
 423     ULONG_PTR Context
 424 );
 425
 426 INIT_FUNCTION
 427 VOID
 428 NTAPI
 429 ExpInitializeHandleTables(
 430     VOID
 431 );
 432
 433 PHANDLE_TABLE
 434 NTAPI
 435 ExCreateHandleTable(
 436     IN PEPROCESS Process OPTIONAL
 437 );
 438
 439 VOID
 440 NTAPI
 441 ExUnlockHandleTableEntry(
 442     IN PHANDLE_TABLE HandleTable,
 443     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 444 );
 445
 446 HANDLE
 447 NTAPI
 448 ExCreateHandle(
 449     IN PHANDLE_TABLE HandleTable,
 450     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 451 );
 452
 453 VOID
 454 NTAPI
 455 ExDestroyHandleTable(
 456     IN PHANDLE_TABLE HandleTable,
 457     IN PVOID DestroyHandleProcedure OPTIONAL
 458 );
 459
 460 BOOLEAN
 461 NTAPI
 462 ExDestroyHandle(
 463     IN PHANDLE_TABLE HandleTable,
 464     IN HANDLE Handle,
 465     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 466 );
 467
 468 PHANDLE_TABLE_ENTRY
 469 NTAPI
 470 ExMapHandleToPointer(
 471     IN PHANDLE_TABLE HandleTable,
 472     IN HANDLE Handle
 473 );
 474
 475 PHANDLE_TABLE
 476 NTAPI
 477 ExDupHandleTable(
 478     IN PEPROCESS Process,
 479     IN PHANDLE_TABLE HandleTable,
 480     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 481     IN ULONG_PTR Mask
 482 );
 483
 484 BOOLEAN
 485 NTAPI
 486 ExChangeHandle(
 487     IN PHANDLE_TABLE HandleTable,
 488     IN HANDLE Handle,
 489     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 490     IN ULONG_PTR Context
 491 );
 492
 493 VOID
 494 NTAPI
 495 ExSweepHandleTable(
 496     IN PHANDLE_TABLE HandleTable,
 497     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 498     IN PVOID Context
 499 );
 500
 501 PHANDLE_TABLE_ENTRY
 502 NTAPI
 503 ExpLookupHandleTableEntry(
 504     IN PHANDLE_TABLE HandleTable,
 505     IN EXHANDLE Handle
 506 );
 507
 508 BOOLEAN
 509 NTAPI
 510 ExpLockHandleTableEntry(
 511     IN PHANDLE_TABLE HandleTable,
 512     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 513 );
 514
 515 /* PSEH EXCEPTION HANDLING **************************************************/
 516
 517 LONG
 518 NTAPI
 519 ExSystemExceptionFilter(VOID);
 520
 521 /* CALLBACKS *****************************************************************/
 522
 523 FORCEINLINE
 524 VOID
 525 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 526              IN PVOID Context,
 527              IN PVOID Argument1,
 528              IN PVOID Argument2)
 529 {
 530     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 531     PEX_CALLBACK_FUNCTION Function;
 532
 533     /* Reference the block */
 534     CallbackBlock = ExReferenceCallBackBlock(Callback);
 535     if (CallbackBlock)
 536     {
 537         /* Get the function */
 538         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 539
 540         /* Do the callback */
 541         Function(Context, Argument1, Argument2);
 542
 543         /* Now dereference it */
 544         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 545     }
 546 }
 547
 548 /* FAST REFS ******************************************************************/
 549
 550 FORCEINLINE
 551 PVOID
 552 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 553 {
 554     /* Return the unbiased pointer */
 555     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 556 }
 557
 558 FORCEINLINE
 559 ULONG
 560 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 561 {
 562     /* Return the reference count */
 563     return (ULONG)FastRef.RefCnt;
 564 }
 565
 566 FORCEINLINE
 567 VOID
 568 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 569                           IN OPTIONAL PVOID Object)
 570 {
 571     /* Sanity check */
 572     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 573
 574     /* Check if an object is being set */
 575     if (!Object)
 576     {
 577         /* Clear the field */
 578         FastRef->Object = NULL;
 579     }
 580     else
 581     {
 582         /* Otherwise, we assume the object was referenced and is ready */
 583         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 584     }
 585 }
 586
 587 FORCEINLINE
 588 EX_FAST_REF
 589 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 590 {
 591     EX_FAST_REF OldValue, NewValue;
 592
 593     /* Start reference loop */
 594     for (;;)
 595     {
 596         /* Get the current reference count */
 597         OldValue = *FastRef;
 598         if (OldValue.RefCnt)
 599         {
 600             /* Increase the reference count */
 601             NewValue.Value = OldValue.Value - 1;
 602             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 603                                                 NewValue.Object,
 604                                                 OldValue.Object);
 605             if (NewValue.Object != OldValue.Object) continue;
 606         }
 607
 608         /* We are done */
 609         break;
 610     }
 611
 612     /* Return the old value */
 613     return OldValue;
 614 }
 615
 616 FORCEINLINE
 617 BOOLEAN
 618 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 619                       IN PVOID Object)
 620 {
 621     EX_FAST_REF OldValue, NewValue;
 622
 623     /* Sanity checks */
 624     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 625
 626     /* Start update loop */
 627     for (;;)
 628     {
 629         /* Get the current reference count */
 630         OldValue = *FastRef;
 631
 632         /* Check if the current count is too high or if the pointer changed */
 633         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 634             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 635         {
 636             /* Fail */
 637             return FALSE;
 638         }
 639
 640         /* Update the reference count */
 641         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 642         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 643                                             NewValue.Object,
 644                                             OldValue.Object);
 645         if (NewValue.Object != OldValue.Object) continue;
 646
 647         /* We are done */
 648         break;
 649     }
 650
 651     /* Return success */
 652     return TRUE;
 653 }
 654
 655 FORCEINLINE
 656 BOOLEAN
 657 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 658                        IN PVOID Object)
 659 {
 660     EX_FAST_REF OldValue, NewValue;
 661
 662     /* Sanity checks */
 663     ASSERT(Object != NULL);
 664     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 665
 666     /* Start reference loop */
 667     for (;;)
 668     {
 669         /* Get the current reference count */
 670         OldValue = *FastRef;
 671
 672         /* Check if we're full if if the pointer changed */
 673         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 674
 675         /* Decrease the reference count */
 676         NewValue.Value = OldValue.Value + 1;
 677         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 678                                             NewValue.Object,
 679                                             OldValue.Object);
 680         if (NewValue.Object != OldValue.Object) continue;
 681
 682         /* We are done */
 683         break;
 684     }
 685
 686     /* Return success */
 687     return TRUE;
 688 }
 689
 690 FORCEINLINE
 691 EX_FAST_REF
 692 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 693                     IN PVOID Object)
 694 {
 695     EX_FAST_REF NewValue, OldValue;
 696
 697     /* Sanity check */
 698     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 699
 700     /* Check if an object is being set */
 701     if (!Object)
 702     {
 703         /* Clear the field */
 704         NewValue.Object = NULL;
 705     }
 706     else
 707     {
 708         /* Otherwise, we assume the object was referenced and is ready */
 709         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 710     }
 711
 712     /* Update the object */
 713     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 714     return OldValue;
 715 }
 716
 717 FORCEINLINE
 718 EX_FAST_REF
 719 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 720                            IN PVOID Object,
 721                            IN PVOID OldObject)
 722 {
 723     EX_FAST_REF OldValue, NewValue;
 724
 725     /* Sanity check and start swap loop */
 726     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 727     for (;;)
 728     {
 729         /* Get the current value */
 730         OldValue = *FastRef;
 731
 732         /* Make sure there's enough references to swap */
 733         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 734
 735         /* Check if we have an object to swap */
 736         if (Object)
 737         {
 738             /* Set up the value with maximum fast references */
 739             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 740         }
 741         else
 742         {
 743             /* Write the object address itself (which is empty) */
 744             NewValue.Value = (ULONG_PTR)Object;
 745         }
 746
 747         /* Do the actual compare exchange */
 748         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 749                                             NewValue.Object,
 750                                             OldValue.Object);
 751         if (NewValue.Object != OldValue.Object) continue;
 752
 753         /* All done */
 754         break;
 755     }
 756
 757     /* Return the old value */
 758     return OldValue;
 759 }
 760
 761 /* RUNDOWN *******************************************************************/
 762
 763 FORCEINLINE
 764 PEX_RUNDOWN_REF
 765 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 766                              IN ULONG ProcNumber)
 767 {
 768     return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 769                              RunRefCacheAware->RunRefSize *
 770                              (ProcNumber % RunRefCacheAware->Number));
 771 }
 772
 773 /*++
 774  * @name ExfAcquireRundownProtection
 775  * INTERNAL MACRO
 776  *
 777  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 778  *     the specified descriptor.
 779  *
 780  * @param RunRef
 781  *        Pointer to a rundown reference descriptor.
 782  *
 783  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 784  *
 785  * @remarks This is the internal macro for system use only.In case the rundown
 786  *          was active, then the slow-path will be called through the exported
 787  *          function.
 788  *
 789  *--*/
 790 FORCEINLINE
 791 BOOLEAN
 792 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 793 {
 794     ULONG_PTR Value, NewValue;
 795
 796     /* Get the current value and mask the active bit */
 797     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 798
 799     /* Add a reference */
 800     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 801
 802     /* Change the value */
 803     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 804     if (NewValue != Value)
 805     {
 806         /* Rundown was active, use long path */
 807         return ExfAcquireRundownProtection(RunRef);
 808     }
 809
 810     /* Success */
 811     return TRUE;
 812 }
 813
 814 /*++
 815  * @name ExReleaseRundownProtection
 816  * INTERNAL MACRO
 817  *
 818  *     The ExReleaseRundownProtection routine releases rundown protection for
 819  *     the specified descriptor.
 820  *
 821  * @param RunRef
 822  *        Pointer to a rundown reference descriptor.
 823  *
 824  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 825  *
 826  * @remarks This is the internal macro for system use only.In case the rundown
 827  *          was active, then the slow-path will be called through the exported
 828  *          function.
 829  *
 830  *--*/
 831 FORCEINLINE
 832 VOID
 833 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 834 {
 835     ULONG_PTR Value, NewValue;
 836
 837     /* Get the current value and mask the active bit */
 838     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 839
 840     /* Remove a reference */
 841     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 842
 843     /* Change the value */
 844     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 845
 846     /* Check if the rundown was active */
 847     if (NewValue != Value)
 848     {
 849         /* Rundown was active, use long path */
 850         ExfReleaseRundownProtection(RunRef);
 851     }
 852     else
 853     {
 854         /* Sanity check */
 855         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 856     }
 857 }
 858
 859 /*++
 860  * @name ExInitializeRundownProtection
 861  * INTERNAL MACRO
 862  *
 863  *     The ExInitializeRundownProtection routine initializes a rundown
 864  *     protection descriptor.
 865  *
 866  * @param RunRef
 867  *        Pointer to a rundown reference descriptor.
 868  *
 869  * @return None.
 870  *
 871  * @remarks This is the internal macro for system use only.
 872  *
 873  *--*/
 874 FORCEINLINE
 875 VOID
 876 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 877 {
 878     /* Set the count to zero */
 879     RunRef->Count = 0;
 880 }
 881
 882 /*++
 883  * @name ExWaitForRundownProtectionRelease
 884  * INTERNAL MACRO
 885  *
 886  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 887  *     rundown descriptor has been released.
 888  *
 889  * @param RunRef
 890  *        Pointer to a rundown reference descriptor.
 891  *
 892  * @return None.
 893  *
 894  * @remarks This is the internal macro for system use only. If a wait is actually
 895  *          necessary, then the slow path is taken through the exported function.
 896  *
 897  *--*/
 898 FORCEINLINE
 899 VOID
 900 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 901 {
 902     ULONG_PTR Value;
 903
 904     /* Set the active bit */
 905     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 906     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 907     {
 908         /* If the the rundown wasn't already active, then take the long path */
 909         ExfWaitForRundownProtectionRelease(RunRef);
 910     }
 911 }
 912
 913 /*++
 914  * @name ExRundownCompleted
 915  * INTERNAL MACRO
 916  *
 917  *     The ExRundownCompleted routine completes the rundown of the specified
 918  *     descriptor by setting the active bit.
 919  *
 920  * @param RunRef
 921  *        Pointer to a rundown reference descriptor.
 922  *
 923  * @return None.
 924  *
 925  * @remarks This is the internal macro for system use only.
 926  *
 927  *--*/
 928 FORCEINLINE
 929 VOID
 930 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 931 {
 932     /* Sanity check */
 933     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 934
 935     /* Mark the counter as active */
 936     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 937 }
 938
 939 /* PUSHLOCKS *****************************************************************/
 940
 941 /* FIXME: VERIFY THESE! */
 942
 943 VOID
 944 FASTCALL
 945 ExBlockPushLock(
 946     IN PEX_PUSH_LOCK PushLock,
 947     IN PVOID WaitBlock
 948 );
 949
 950 VOID
 951 FASTCALL
 952 ExfUnblockPushLock(
 953     IN PEX_PUSH_LOCK PushLock,
 954     IN PVOID CurrentWaitBlock
 955 );
 956
 957 VOID
 958 FASTCALL
 959 ExWaitForUnblockPushLock(
 960     IN PEX_PUSH_LOCK PushLock,
 961     IN PVOID WaitBlock
 962 );
 963
 964 /*++
 965  * @name _ExInitializePushLock
 966  * INTERNAL MACRO
 967  *
 968  *     The _ExInitializePushLock macro initializes a PushLock.
 969  *
 970  * @params PushLock
 971  *         Pointer to the pushlock which is to be initialized.
 972  *
 973  * @return None.
 974  *
 975  * @remarks None.
 976  *
 977  *--*/
 978 FORCEINLINE
 979 VOID
 980 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 981 {
 982     /* Set the value to 0 */
 983     PushLock->Ptr = 0;
 984 }
 985 #define ExInitializePushLock _ExInitializePushLock
 986
 987 /*++
 988  * @name ExAcquirePushLockExclusive
 989  * INTERNAL MACRO
 990  *
 991  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 992  *
 993  * @params PushLock
 994  *         Pointer to the pushlock which is to be acquired.
 995  *
 996  * @return None.
 997  *
 998  * @remarks The function attempts the quickest route to acquire the lock, which is
 999  *          to simply set the lock bit.
1000  *          However, if the pushlock is already shared, the slower path is taken.
1001  *
1002  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1003  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1004  *
1005  *--*/
1006 FORCEINLINE
1007 VOID
1008 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1009 {
1010     /* Try acquiring the lock */
1011     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1012     {
1013         /* Someone changed it, use the slow path */
1014         ExfAcquirePushLockExclusive(PushLock);
1015     }
1016
1017     /* Sanity check */
1018     ASSERT(PushLock->Locked);
1019 }
1020
1021 /*++
1022 * @name ExTryToAcquirePushLockExclusive
1023 * INTERNAL MACRO
1024 *
1025 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1026 *
1027 * @params PushLock
1028 *         Pointer to the pushlock which is to be acquired.
1029 *
1030 * @return None.
1031 *
1032 * @remarks The function attempts the quickest route to acquire the lock, which is
1033 *          to simply set the lock bit.
1034 *          However, if the pushlock is already shared, the slower path is taken.
1035 *
1036 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1037 *          This macro should usually be paired up with KeAcquireCriticalRegion.
1038 *
1039 *--*/
1040 FORCEINLINE
1041 BOOLEAN
1042 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1043 {
1044     /* Try acquiring the lock */
1045     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1046     {
1047         /* Can't acquire */
1048         return FALSE;
1049     }
1050
1051     /* Got acquired */
1052     ASSERT (PushLock->Locked);
1053     return TRUE;
1054 }
1055
1056 /*++
1057  * @name ExAcquirePushLockShared
1058  * INTERNAL MACRO
1059  *
1060  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1061  *
1062  * @params PushLock
1063  *         Pointer to the pushlock which is to be acquired.
1064  *
1065  * @return None.
1066  *
1067  * @remarks The function attempts the quickest route to acquire the lock, which is
1068  *          to simply set the lock bit and set the share count to one.
1069  *          However, if the pushlock is already shared, the slower path is taken.
1070  *
1071  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1072  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1073  *
1074  *--*/
1075 FORCEINLINE
1076 VOID
1077 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1078 {
1079     EX_PUSH_LOCK NewValue;
1080
1081     /* Try acquiring the lock */
1082     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1083     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1084     {
1085         /* Someone changed it, use the slow path */
1086         ExfAcquirePushLockShared(PushLock);
1087     }
1088
1089     /* Sanity checks */
1090     ASSERT(PushLock->Locked);
1091 }
1092
1093 /*++
1094  * @name ExConvertPushLockSharedToExclusive
1095  * INTERNAL MACRO
1096  *
1097  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1098  *     pushlock to a shared pushlock.
1099  *
1100  * @params PushLock
1101  *         Pointer to the pushlock which is to be converted.
1102  *
1103  * @return FALSE if conversion failed, TRUE otherwise.
1104  *
1105  * @remarks The function attempts the quickest route to convert the lock, which is
1106  *          to simply set the lock bit and remove any other bits.
1107  *
1108  *--*/
1109 FORCEINLINE
1110 BOOLEAN
1111 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1112 {
1113     EX_PUSH_LOCK OldValue;
1114
1115     /* Set the expected old value */
1116     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1117
1118     /* Try converting the lock */
1119     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1120         OldValue.Ptr)
1121     {
1122         /* Conversion failed */
1123         return FALSE;
1124     }
1125
1126     /* Sanity check */
1127     ASSERT(PushLock->Locked);
1128     return TRUE;
1129 }
1130
1131 /*++
1132  * @name ExWaitOnPushLock
1133  * INTERNAL MACRO
1134  *
1135  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1136  *
1137  * @params PushLock
1138  *         Pointer to a pushlock.
1139  *
1140  * @return None.
1141  *
1142  * @remarks The function attempts to get any exclusive waiters out of their slow
1143  *          path by forcing an instant acquire/release operation.
1144  *
1145  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1146  *
1147  *--*/
1148 FORCEINLINE
1149 VOID
1150 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1151 {
1152     /* Check if we're locked */
1153     if (PushLock->Locked)
1154     {
1155         /* Acquire the lock */
1156         ExfAcquirePushLockExclusive(PushLock);
1157         ASSERT(PushLock->Locked);
1158
1159         /* Release it */
1160         ExfReleasePushLockExclusive(PushLock);
1161     }
1162 }
1163
1164 /*++
1165  * @name ExReleasePushLockShared
1166  * INTERNAL MACRO
1167  *
1168  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1169  *
1170  * @params PushLock
1171  *         Pointer to a previously acquired pushlock.
1172  *
1173  * @return None.
1174  *
1175  * @remarks The function attempts the quickest route to release the lock, which is
1176  *          to simply decrease the share count and remove the lock bit.
1177  *          However, if the pushlock is being waited on then the long path is taken.
1178  *
1179  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1180  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1181  *
1182  *--*/
1183 FORCEINLINE
1184 VOID
1185 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1186 {
1187     EX_PUSH_LOCK OldValue;
1188
1189     /* Sanity checks */
1190     ASSERT(PushLock->Locked);
1191
1192     /* Try to clear the pushlock */
1193     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1194     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1195     {
1196         /* There are still other people waiting on it */
1197         ExfReleasePushLockShared(PushLock);
1198     }
1199 }
1200
1201 /*++
1202  * @name ExReleasePushLockExclusive
1203  * INTERNAL MACRO
1204  *
1205  *     The ExReleasePushLockExclusive macro releases a previously
1206  *     exclusively acquired PushLock.
1207  *
1208  * @params PushLock
1209  *         Pointer to a previously acquired pushlock.
1210  *
1211  * @return None.
1212  *
1213  * @remarks The function attempts the quickest route to release the lock, which is
1214  *          to simply clear the locked bit.
1215  *          However, if the pushlock is being waited on, the slow path is taken
1216  *          in an attempt to wake up the lock.
1217  *
1218  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1219  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1220  *
1221  *--*/
1222 FORCEINLINE
1223 VOID
1224 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1225 {
1226     EX_PUSH_LOCK OldValue;
1227
1228     /* Sanity checks */
1229     ASSERT(PushLock->Locked);
1230
1231     /* Unlock the pushlock */
1232     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1233                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1234
1235     /* Sanity checks */
1236     ASSERT(OldValue.Locked);
1237     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1238
1239     /* Check if anyone is waiting on it and it's not already waking*/
1240     if ((OldValue.Waiting) && !(OldValue.Waking))
1241     {
1242         /* Wake it up */
1243         ExfTryToWakePushLock(PushLock);
1244     }
1245 }
1246
1247 /*++
1248  * @name ExReleasePushLock
1249  * INTERNAL MACRO
1250  *
1251  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1252  *
1253  * @params PushLock
1254  *         Pointer to a previously acquired pushlock.
1255  *
1256  * @return None.
1257  *
1258  * @remarks The function attempts the quickest route to release the lock, which is
1259  *          to simply clear all the fields and decrease the share count if required.
1260  *          However, if the pushlock is being waited on then the long path is taken.
1261  *
1262  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1263  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1264  *
1265  *--*/
1266 FORCEINLINE
1267 VOID
1268 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1269 {
1270     EX_PUSH_LOCK OldValue = *PushLock;
1271     EX_PUSH_LOCK NewValue;
1272
1273     /* Sanity checks */
1274     ASSERT(OldValue.Locked);
1275
1276     /* Check if the pushlock is shared */
1277     if (OldValue.Shared > 1)
1278     {
1279         /* Decrease the share count */
1280         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1281     }
1282     else
1283     {
1284         /* Clear the pushlock entirely */
1285         NewValue.Value = 0;
1286     }
1287
1288     /* Check if nobody is waiting on us and try clearing the lock here */
1289     if ((OldValue.Waiting) ||
1290         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1291          OldValue.Ptr))
1292     {
1293         /* We have waiters, use the long path */
1294         ExfReleasePushLock(PushLock);
1295     }
1296 }
1297
1298 /* FAST MUTEX INLINES *********************************************************/
1299
1300 FORCEINLINE
1301 VOID
1302 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1303 {
1304     PKTHREAD Thread = KeGetCurrentThread();
1305
1306     /* Sanity check */
1307     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1308            (Thread->CombinedApcDisable != 0) ||
1309            (Thread->Teb == NULL) ||
1310            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1311     ASSERT(FastMutex->Owner != Thread);
1312
1313     /* Decrease the count */
1314     if (InterlockedDecrement(&FastMutex->Count))
1315     {
1316         /* Someone is still holding it, use slow path */
1317         KiAcquireFastMutex(FastMutex);
1318     }
1319
1320     /* Set the owner */
1321     FastMutex->Owner = Thread;
1322 }
1323
1324 FORCEINLINE
1325 VOID
1326 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1327 {
1328     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1329            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1330            (KeGetCurrentThread()->Teb == NULL) ||
1331            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1332     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1333
1334     /* Erase the owner */
1335     FastMutex->Owner = NULL;
1336
1337     /* Increase the count */
1338     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1339     {
1340         /* Someone was waiting for it, signal the waiter */
1341         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1342     }
1343 }
1344
1345 FORCEINLINE
1346 VOID
1347 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1348 {
1349     KIRQL OldIrql;
1350     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1351
1352     /* Raise IRQL to APC */
1353     KeRaiseIrql(APC_LEVEL, &OldIrql);
1354
1355     /* Decrease the count */
1356     if (InterlockedDecrement(&FastMutex->Count))
1357     {
1358         /* Someone is still holding it, use slow path */
1359         KiAcquireFastMutex(FastMutex);
1360     }
1361
1362     /* Set the owner and IRQL */
1363     FastMutex->Owner = KeGetCurrentThread();
1364     FastMutex->OldIrql = OldIrql;
1365 }
1366
1367 FORCEINLINE
1368 VOID
1369 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1370 {
1371     KIRQL OldIrql;
1372     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1373
1374     /* Erase the owner */
1375     FastMutex->Owner = NULL;
1376     OldIrql = (KIRQL)FastMutex->OldIrql;
1377
1378     /* Increase the count */
1379     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1380     {
1381         /* Someone was waiting for it, signal the waiter */
1382         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1383     }
1384
1385     /* Lower IRQL back */
1386     KeLowerIrql(OldIrql);
1387 }
1388
1389 FORCEINLINE
1390 BOOLEAN
1391 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1392 {
1393     KIRQL OldIrql;
1394     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1395
1396     /* Raise to APC_LEVEL */
1397     KeRaiseIrql(APC_LEVEL, &OldIrql);
1398
1399     /* Check if we can quickly acquire it */
1400     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1401     {
1402         /* We have, set us as owners */
1403         FastMutex->Owner = KeGetCurrentThread();
1404         FastMutex->OldIrql = OldIrql;
1405         return TRUE;
1406     }
1407     else
1408     {
1409         /* Acquire attempt failed */
1410         KeLowerIrql(OldIrql);
1411         YieldProcessor();
1412         return FALSE;
1413     }
1414 }
1415
1416 FORCEINLINE
1417 VOID
1418 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1419 {
1420     /* Enter the Critical Region */
1421     KeEnterCriticalRegion();
1422
1423     /* Acquire the mutex unsafely */
1424     _ExAcquireFastMutexUnsafe(FastMutex);
1425 }
1426
1427 FORCEINLINE
1428 VOID
1429 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1430 {
1431     /* Release the mutex unsafely */
1432     _ExReleaseFastMutexUnsafe(FastMutex);
1433
1434     /* Leave the critical region */
1435     KeLeaveCriticalRegion();
1436 }
1437
1438 /* OTHER FUNCTIONS **********************************************************/
1439
1440 BOOLEAN
1441 NTAPI
1442 ExTryToAcquireResourceExclusiveLite(
1443     IN PERESOURCE Resource
1444 );
1445
1446 NTSTATUS
1447 ExpSetTimeZoneInformation(
1448     IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1449 );
1450
1451 BOOLEAN
1452 NTAPI
1453 ExAcquireTimeRefreshLock(
1454     IN BOOLEAN Wait
1455 );
1456
1457 VOID
1458 NTAPI
1459 ExReleaseTimeRefreshLock(
1460     VOID
1461 );
1462
1463 VOID
1464 NTAPI
1465 ExUpdateSystemTimeFromCmos(
1466     IN BOOLEAN UpdateInterruptTime,
1467     IN ULONG MaxSepInSeconds
1468 );
1469
1470 VOID
1471 NTAPI
1472 ExAllocateLocallyUniqueId(
1473     OUT LUID *LocallyUniqueId
1474 );
1475
1476 VOID
1477 NTAPI
1478 ExTimerRundown(
1479     VOID
1480 );
1481
1482 INIT_FUNCTION
1483 VOID
1484 NTAPI
1485 HeadlessInit(
1486     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1487 );
1488
1489 INIT_FUNCTION
1490 VOID
1491 NTAPI
1492 XIPInit(
1493     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1494 );
1495
1496 #define InterlockedDecrementUL(Addend) \
1497    (ULONG)InterlockedDecrement((PLONG)(Addend))
1498
1499 #define InterlockedIncrementUL(Addend) \
1500    (ULONG)InterlockedIncrement((PLONG)(Addend))
1501
1502 #define InterlockedExchangeUL(Target, Value) \
1503    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1504
1505 #define InterlockedExchangeAddUL(Addend, Value) \
1506    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1507
1508 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1509    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1510
1511 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1512    (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1513
1514 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1515    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))