ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31 extern ULONG ExCriticalWorkerThreads;
  32 extern ULONG ExDelayedWorkerThreads;
  33
  34 extern PVOID ExpDefaultErrorPort;
  35 extern PEPROCESS ExpDefaultErrorPortProcess;
  36
  37 /*
  38  * NT/Cm Version Info variables
  39  */
  40 extern ULONG NtMajorVersion;
  41 extern ULONG NtMinorVersion;
  42 extern ULONG NtBuildNumber;
  43 extern ULONG CmNtSpBuildNumber;
  44 extern ULONG CmNtCSDVersion;
  45 extern ULONG CmNtCSDReleaseType;
  46 extern UNICODE_STRING CmVersionString;
  47 extern UNICODE_STRING CmCSDVersionString;
  48 extern CHAR NtBuildLab[];
  49
  50 // #ifdef _WINKD_
  51 /*
  52  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  53  */
  54 typedef enum _WINKD_WORKER_STATE
  55 {
  56     WinKdWorkerReady = 0,
  57     WinKdWorkerStart,
  58     WinKdWorkerInitialized
  59 } WINKD_WORKER_STATE;
  60
  61 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  62 extern WINKD_WORKER_STATE ExpDebuggerWork;
  63 extern PEPROCESS ExpDebuggerProcessAttach;
  64 extern PEPROCESS ExpDebuggerProcessKill;
  65 extern ULONG_PTR ExpDebuggerPageIn;
  66
  67 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  68 // #endif /* _WINKD_ */
  69
  70 #ifdef _WIN64
  71 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  72 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  73 #else
  74 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  75 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  76 #endif
  77 #define HANDLE_TAG_BITS (2)
  78 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
  79 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
  80
  81 typedef union _EXHANDLE
  82 {
  83      struct
  84      {
  85          ULONG_PTR TagBits:     HANDLE_TAG_BITS;
  86          ULONG_PTR Index:       HANDLE_INDEX_BITS;
  87          ULONG_PTR KernelFlag : KERNEL_FLAG_BITS;
  88      };
  89      struct
  90      {
  91          ULONG_PTR TagBits2:    HANDLE_TAG_BITS;
  92          ULONG_PTR LowIndex:    HANDLE_LOW_BITS;
  93          ULONG_PTR MidIndex:    HANDLE_HIGH_BITS;
  94          ULONG_PTR HighIndex:   HANDLE_HIGH_BITS;
  95          ULONG_PTR KernelFlag2: KERNEL_FLAG_BITS;
  96      };
  97      HANDLE GenericHandleOverlay;
  98      ULONG_PTR Value;
  99      ULONG AsULONG;
 100 } EXHANDLE, *PEXHANDLE;
 101
 102 typedef struct _ETIMER
 103 {
 104     KTIMER KeTimer;
 105     KAPC TimerApc;
 106     KDPC TimerDpc;
 107     LIST_ENTRY ActiveTimerListEntry;
 108     KSPIN_LOCK Lock;
 109     LONG Period;
 110     BOOLEAN ApcAssociated;
 111     BOOLEAN WakeTimer;
 112     LIST_ENTRY WakeTimerListEntry;
 113 } ETIMER, *PETIMER;
 114
 115 typedef struct
 116 {
 117     PCALLBACK_OBJECT *CallbackObject;
 118     PWSTR Name;
 119 } SYSTEM_CALLBACKS;
 120
 121 typedef struct _HARDERROR_USER_PARAMETERS
 122 {
 123     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 124     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 125     WCHAR Buffer[ANYSIZE_ARRAY];
 126 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 127
 128 #define MAX_FAST_REFS           7
 129
 130 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 131 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 132 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 133 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 134 #define ExRundownCompleted                              _ExRundownCompleted
 135 #define ExGetPreviousMode                               KeGetPreviousMode
 136
 137
 138 //
 139 // Various bits tagged on the handle or handle table
 140 //
 141 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 142 #define FREE_HANDLE_MASK                -1
 143
 144 //
 145 // Number of entries in each table level
 146 //
 147 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 148 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 149 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 150
 151 //
 152 // Maximum index in each table level before we need another table
 153 //
 154 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 155 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 156 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 157
 158 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 159 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 160 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 161
 162 NTSTATUS
 163 NTAPI
 164 ExGetPoolTagInfo(
 165     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 166     IN ULONG SystemInformationLength,
 167     IN OUT PULONG ReturnLength OPTIONAL
 168 );
 169
 170 typedef struct _UUID_CACHED_VALUES_STRUCT
 171 {
 172     ULONGLONG Time;
 173     LONG AllocatedCount;
 174     UCHAR ClockSeqHiAndReserved;
 175     UCHAR ClockSeqLow;
 176     UCHAR NodeId [ 6 ] ;
 177 } UUID_CACHED_VALUES_STRUCT, *PUUID_CACHED_VALUES_STRUCT;
 178
 179 /* INITIALIZATION FUNCTIONS *************************************************/
 180
 181 INIT_FUNCTION
 182 BOOLEAN
 183 NTAPI
 184 ExpWin32kInit(VOID);
 185
 186 VOID
 187 NTAPI
 188 ExInit2(VOID);
 189
 190 VOID
 191 NTAPI
 192 Phase1Initialization(
 193     IN PVOID Context
 194 );
 195
 196 INIT_FUNCTION
 197 VOID
 198 NTAPI
 199 ExpInitializePushLocks(VOID);
 200
 201 BOOLEAN
 202 NTAPI
 203 ExRefreshTimeZoneInformation(
 204     IN PLARGE_INTEGER SystemBootTime
 205 );
 206
 207 INIT_FUNCTION
 208 VOID
 209 NTAPI
 210 ExpInitializeWorkerThreads(VOID);
 211
 212 VOID
 213 NTAPI
 214 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 215
 216 INIT_FUNCTION
 217 VOID
 218 NTAPI
 219 ExpInitLookasideLists(VOID);
 220
 221 INIT_FUNCTION
 222 VOID
 223 NTAPI
 224 ExInitializeSystemLookasideList(
 225     IN PGENERAL_LOOKASIDE List,
 226     IN POOL_TYPE Type,
 227     IN ULONG Size,
 228     IN ULONG Tag,
 229     IN USHORT MaximumDepth,
 230     IN PLIST_ENTRY ListHead
 231 );
 232
 233 INIT_FUNCTION
 234 BOOLEAN
 235 NTAPI
 236 ExpInitializeCallbacks(VOID);
 237
 238 INIT_FUNCTION
 239 BOOLEAN
 240 NTAPI
 241 ExpUuidInitialization(VOID);
 242
 243 INIT_FUNCTION
 244 BOOLEAN
 245 NTAPI
 246 ExLuidInitialization(VOID);
 247
 248 INIT_FUNCTION
 249 VOID
 250 NTAPI
 251 ExpInitializeExecutive(
 252     IN ULONG Cpu,
 253     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 254 );
 255
 256 VOID
 257 NTAPI
 258 ExShutdownSystem(VOID);
 259
 260 INIT_FUNCTION
 261 BOOLEAN
 262 NTAPI
 263 ExpInitializeEventImplementation(VOID);
 264
 265 INIT_FUNCTION
 266 BOOLEAN
 267 NTAPI
 268 ExpInitializeKeyedEventImplementation(VOID);
 269
 270 INIT_FUNCTION
 271 BOOLEAN
 272 NTAPI
 273 ExpInitializeEventPairImplementation(VOID);
 274
 275 INIT_FUNCTION
 276 BOOLEAN
 277 NTAPI
 278 ExpInitializeSemaphoreImplementation(VOID);
 279
 280 INIT_FUNCTION
 281 BOOLEAN
 282 NTAPI
 283 ExpInitializeMutantImplementation(VOID);
 284
 285 INIT_FUNCTION
 286 BOOLEAN
 287 NTAPI
 288 ExpInitializeTimerImplementation(VOID);
 289
 290 INIT_FUNCTION
 291 BOOLEAN
 292 NTAPI
 293 ExpInitializeProfileImplementation(VOID);
 294
 295 INIT_FUNCTION
 296 VOID
 297 NTAPI
 298 ExpResourceInitialization(VOID);
 299
 300 INIT_FUNCTION
 301 VOID
 302 NTAPI
 303 ExInitPoolLookasidePointers(VOID);
 304
 305 /* Callback Functions ********************************************************/
 306
 307 VOID
 308 NTAPI
 309 ExInitializeCallBack(
 310     IN OUT PEX_CALLBACK Callback
 311 );
 312
 313 PEX_CALLBACK_ROUTINE_BLOCK
 314 NTAPI
 315 ExAllocateCallBack(
 316     IN PEX_CALLBACK_FUNCTION Function,
 317     IN PVOID Context
 318 );
 319
 320 VOID
 321 NTAPI
 322 ExFreeCallBack(
 323     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 324 );
 325
 326 BOOLEAN
 327 NTAPI
 328 ExCompareExchangeCallBack (
 329     IN OUT PEX_CALLBACK CallBack,
 330     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 331     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 332 );
 333
 334 PEX_CALLBACK_ROUTINE_BLOCK
 335 NTAPI
 336 ExReferenceCallBackBlock(
 337     IN OUT PEX_CALLBACK CallBack
 338 );
 339
 340 VOID
 341 NTAPI
 342 ExDereferenceCallBackBlock(
 343     IN OUT PEX_CALLBACK CallBack,
 344     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 345 );
 346
 347 PEX_CALLBACK_FUNCTION
 348 NTAPI
 349 ExGetCallBackBlockRoutine(
 350     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 351 );
 352
 353 PVOID
 354 NTAPI
 355 ExGetCallBackBlockContext(
 356     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 357 );
 358
 359 VOID
 360 NTAPI
 361 ExWaitForCallBacks(
 362     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 363 );
 364
 365 /* Rundown Functions ********************************************************/
 366
 367 VOID
 368 FASTCALL
 369 ExfInitializeRundownProtection(
 370      OUT PEX_RUNDOWN_REF RunRef
 371 );
 372
 373 VOID
 374 FASTCALL
 375 ExfReInitializeRundownProtection(
 376      OUT PEX_RUNDOWN_REF RunRef
 377 );
 378
 379 BOOLEAN
 380 FASTCALL
 381 ExfAcquireRundownProtection(
 382      IN OUT PEX_RUNDOWN_REF RunRef
 383 );
 384
 385 BOOLEAN
 386 FASTCALL
 387 ExfAcquireRundownProtectionEx(
 388      IN OUT PEX_RUNDOWN_REF RunRef,
 389      IN ULONG Count
 390 );
 391
 392 VOID
 393 FASTCALL
 394 ExfReleaseRundownProtection(
 395      IN OUT PEX_RUNDOWN_REF RunRef
 396 );
 397
 398 VOID
 399 FASTCALL
 400 ExfReleaseRundownProtectionEx(
 401      IN OUT PEX_RUNDOWN_REF RunRef,
 402      IN ULONG Count
 403 );
 404
 405 VOID
 406 FASTCALL
 407 ExfRundownCompleted(
 408      OUT PEX_RUNDOWN_REF RunRef
 409 );
 410
 411 VOID
 412 FASTCALL
 413 ExfWaitForRundownProtectionRelease(
 414      IN OUT PEX_RUNDOWN_REF RunRef
 415 );
 416
 417 /* HANDLE TABLE FUNCTIONS ***************************************************/
 418
 419 typedef BOOLEAN
 420 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 421     PHANDLE_TABLE_ENTRY HandleTableEntry,
 422     HANDLE Handle,
 423     PVOID Context
 424 );
 425
 426 typedef BOOLEAN
 427 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 428     IN PEPROCESS Process,
 429     IN PHANDLE_TABLE HandleTable,
 430     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 431     IN PHANDLE_TABLE_ENTRY NewEntry
 432 );
 433
 434 typedef BOOLEAN
 435 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 436     PHANDLE_TABLE_ENTRY HandleTableEntry,
 437     ULONG_PTR Context
 438 );
 439
 440 INIT_FUNCTION
 441 VOID
 442 NTAPI
 443 ExpInitializeHandleTables(
 444     VOID
 445 );
 446
 447 PHANDLE_TABLE
 448 NTAPI
 449 ExCreateHandleTable(
 450     IN PEPROCESS Process OPTIONAL
 451 );
 452
 453 VOID
 454 NTAPI
 455 ExUnlockHandleTableEntry(
 456     IN PHANDLE_TABLE HandleTable,
 457     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 458 );
 459
 460 HANDLE
 461 NTAPI
 462 ExCreateHandle(
 463     IN PHANDLE_TABLE HandleTable,
 464     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 465 );
 466
 467 VOID
 468 NTAPI
 469 ExDestroyHandleTable(
 470     IN PHANDLE_TABLE HandleTable,
 471     IN PVOID DestroyHandleProcedure OPTIONAL
 472 );
 473
 474 BOOLEAN
 475 NTAPI
 476 ExDestroyHandle(
 477     IN PHANDLE_TABLE HandleTable,
 478     IN HANDLE Handle,
 479     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 480 );
 481
 482 PHANDLE_TABLE_ENTRY
 483 NTAPI
 484 ExMapHandleToPointer(
 485     IN PHANDLE_TABLE HandleTable,
 486     IN HANDLE Handle
 487 );
 488
 489 PHANDLE_TABLE
 490 NTAPI
 491 ExDupHandleTable(
 492     IN PEPROCESS Process,
 493     IN PHANDLE_TABLE HandleTable,
 494     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 495     IN ULONG_PTR Mask
 496 );
 497
 498 BOOLEAN
 499 NTAPI
 500 ExChangeHandle(
 501     IN PHANDLE_TABLE HandleTable,
 502     IN HANDLE Handle,
 503     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 504     IN ULONG_PTR Context
 505 );
 506
 507 VOID
 508 NTAPI
 509 ExSweepHandleTable(
 510     IN PHANDLE_TABLE HandleTable,
 511     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 512     IN PVOID Context
 513 );
 514
 515 PHANDLE_TABLE_ENTRY
 516 NTAPI
 517 ExpLookupHandleTableEntry(
 518     IN PHANDLE_TABLE HandleTable,
 519     IN EXHANDLE Handle
 520 );
 521
 522 BOOLEAN
 523 NTAPI
 524 ExpLockHandleTableEntry(
 525     IN PHANDLE_TABLE HandleTable,
 526     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 527 );
 528
 529 /* PSEH EXCEPTION HANDLING **************************************************/
 530
 531 LONG
 532 NTAPI
 533 ExSystemExceptionFilter(VOID);
 534
 535 /* CALLBACKS *****************************************************************/
 536
 537 FORCEINLINE
 538 VOID
 539 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 540              IN PVOID Context,
 541              IN PVOID Argument1,
 542              IN PVOID Argument2)
 543 {
 544     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 545     PEX_CALLBACK_FUNCTION Function;
 546
 547     /* Reference the block */
 548     CallbackBlock = ExReferenceCallBackBlock(Callback);
 549     if (CallbackBlock)
 550     {
 551         /* Get the function */
 552         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 553
 554         /* Do the callback */
 555         Function(Context, Argument1, Argument2);
 556
 557         /* Now dereference it */
 558         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 559     }
 560 }
 561
 562 /* FAST REFS ******************************************************************/
 563
 564 FORCEINLINE
 565 PVOID
 566 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 567 {
 568     /* Return the unbiased pointer */
 569     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 570 }
 571
 572 FORCEINLINE
 573 ULONG
 574 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 575 {
 576     /* Return the reference count */
 577     return (ULONG)FastRef.RefCnt;
 578 }
 579
 580 FORCEINLINE
 581 VOID
 582 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 583                           IN OPTIONAL PVOID Object)
 584 {
 585     /* Sanity check */
 586     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 587
 588     /* Check if an object is being set */
 589     if (!Object)
 590     {
 591         /* Clear the field */
 592         FastRef->Object = NULL;
 593     }
 594     else
 595     {
 596         /* Otherwise, we assume the object was referenced and is ready */
 597         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 598     }
 599 }
 600
 601 FORCEINLINE
 602 EX_FAST_REF
 603 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 604 {
 605     EX_FAST_REF OldValue, NewValue;
 606
 607     /* Start reference loop */
 608     for (;;)
 609     {
 610         /* Get the current reference count */
 611         OldValue = *FastRef;
 612         if (OldValue.RefCnt)
 613         {
 614             /* Increase the reference count */
 615             NewValue.Value = OldValue.Value - 1;
 616             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 617                                                 NewValue.Object,
 618                                                 OldValue.Object);
 619             if (NewValue.Object != OldValue.Object) continue;
 620         }
 621
 622         /* We are done */
 623         break;
 624     }
 625
 626     /* Return the old value */
 627     return OldValue;
 628 }
 629
 630 FORCEINLINE
 631 BOOLEAN
 632 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 633                       IN PVOID Object)
 634 {
 635     EX_FAST_REF OldValue, NewValue;
 636
 637     /* Sanity checks */
 638     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 639
 640     /* Start update loop */
 641     for (;;)
 642     {
 643         /* Get the current reference count */
 644         OldValue = *FastRef;
 645
 646         /* Check if the current count is too high or if the pointer changed */
 647         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 648             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 649         {
 650             /* Fail */
 651             return FALSE;
 652         }
 653
 654         /* Update the reference count */
 655         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 656         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 657                                             NewValue.Object,
 658                                             OldValue.Object);
 659         if (NewValue.Object != OldValue.Object) continue;
 660
 661         /* We are done */
 662         break;
 663     }
 664
 665     /* Return success */
 666     return TRUE;
 667 }
 668
 669 FORCEINLINE
 670 BOOLEAN
 671 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 672                        IN PVOID Object)
 673 {
 674     EX_FAST_REF OldValue, NewValue;
 675
 676     /* Sanity checks */
 677     ASSERT(Object != NULL);
 678     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 679
 680     /* Start reference loop */
 681     for (;;)
 682     {
 683         /* Get the current reference count */
 684         OldValue = *FastRef;
 685
 686         /* Check if we're full if if the pointer changed */
 687         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 688
 689         /* Decrease the reference count */
 690         NewValue.Value = OldValue.Value + 1;
 691         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 692                                             NewValue.Object,
 693                                             OldValue.Object);
 694         if (NewValue.Object != OldValue.Object) continue;
 695
 696         /* We are done */
 697         break;
 698     }
 699
 700     /* Return success */
 701     return TRUE;
 702 }
 703
 704 FORCEINLINE
 705 EX_FAST_REF
 706 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 707                     IN PVOID Object)
 708 {
 709     EX_FAST_REF NewValue, OldValue;
 710
 711     /* Sanity check */
 712     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 713
 714     /* Check if an object is being set */
 715     if (!Object)
 716     {
 717         /* Clear the field */
 718         NewValue.Object = NULL;
 719     }
 720     else
 721     {
 722         /* Otherwise, we assume the object was referenced and is ready */
 723         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 724     }
 725
 726     /* Update the object */
 727     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 728     return OldValue;
 729 }
 730
 731 FORCEINLINE
 732 EX_FAST_REF
 733 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 734                            IN PVOID Object,
 735                            IN PVOID OldObject)
 736 {
 737     EX_FAST_REF OldValue, NewValue;
 738
 739     /* Sanity check and start swap loop */
 740     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 741     for (;;)
 742     {
 743         /* Get the current value */
 744         OldValue = *FastRef;
 745
 746         /* Make sure there's enough references to swap */
 747         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 748
 749         /* Check if we have an object to swap */
 750         if (Object)
 751         {
 752             /* Set up the value with maximum fast references */
 753             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 754         }
 755         else
 756         {
 757             /* Write the object address itself (which is empty) */
 758             NewValue.Value = (ULONG_PTR)Object;
 759         }
 760
 761         /* Do the actual compare exchange */
 762         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 763                                             NewValue.Object,
 764                                             OldValue.Object);
 765         if (NewValue.Object != OldValue.Object) continue;
 766
 767         /* All done */
 768         break;
 769     }
 770
 771     /* Return the old value */
 772     return OldValue;
 773 }
 774
 775 /* RUNDOWN *******************************************************************/
 776
 777 FORCEINLINE
 778 PEX_RUNDOWN_REF
 779 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
 780                              IN ULONG ProcNumber)
 781 {
 782     return (PEX_RUNDOWN_REF)((ULONG_PTR)RunRefCacheAware->RunRefs +
 783                              RunRefCacheAware->RunRefSize *
 784                              (ProcNumber % RunRefCacheAware->Number));
 785 }
 786
 787 /*++
 788  * @name ExfAcquireRundownProtection
 789  * INTERNAL MACRO
 790  *
 791  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 792  *     the specified descriptor.
 793  *
 794  * @param RunRef
 795  *        Pointer to a rundown reference descriptor.
 796  *
 797  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 798  *
 799  * @remarks This is the internal macro for system use only.In case the rundown
 800  *          was active, then the slow-path will be called through the exported
 801  *          function.
 802  *
 803  *--*/
 804 FORCEINLINE
 805 BOOLEAN
 806 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 807 {
 808     ULONG_PTR Value, NewValue;
 809
 810     /* Get the current value and mask the active bit */
 811     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 812
 813     /* Add a reference */
 814     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 815
 816     /* Change the value */
 817     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 818     if (NewValue != Value)
 819     {
 820         /* Rundown was active, use long path */
 821         return ExfAcquireRundownProtection(RunRef);
 822     }
 823
 824     /* Success */
 825     return TRUE;
 826 }
 827
 828 /*++
 829  * @name ExReleaseRundownProtection
 830  * INTERNAL MACRO
 831  *
 832  *     The ExReleaseRundownProtection routine releases rundown protection for
 833  *     the specified descriptor.
 834  *
 835  * @param RunRef
 836  *        Pointer to a rundown reference descriptor.
 837  *
 838  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 839  *
 840  * @remarks This is the internal macro for system use only.In case the rundown
 841  *          was active, then the slow-path will be called through the exported
 842  *          function.
 843  *
 844  *--*/
 845 FORCEINLINE
 846 VOID
 847 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 848 {
 849     ULONG_PTR Value, NewValue;
 850
 851     /* Get the current value and mask the active bit */
 852     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 853
 854     /* Remove a reference */
 855     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 856
 857     /* Change the value */
 858     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 859
 860     /* Check if the rundown was active */
 861     if (NewValue != Value)
 862     {
 863         /* Rundown was active, use long path */
 864         ExfReleaseRundownProtection(RunRef);
 865     }
 866     else
 867     {
 868         /* Sanity check */
 869         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 870     }
 871 }
 872
 873 /*++
 874  * @name ExInitializeRundownProtection
 875  * INTERNAL MACRO
 876  *
 877  *     The ExInitializeRundownProtection routine initializes a rundown
 878  *     protection descriptor.
 879  *
 880  * @param RunRef
 881  *        Pointer to a rundown reference descriptor.
 882  *
 883  * @return None.
 884  *
 885  * @remarks This is the internal macro for system use only.
 886  *
 887  *--*/
 888 FORCEINLINE
 889 VOID
 890 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 891 {
 892     /* Set the count to zero */
 893     RunRef->Count = 0;
 894 }
 895
 896 /*++
 897  * @name ExWaitForRundownProtectionRelease
 898  * INTERNAL MACRO
 899  *
 900  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 901  *     rundown descriptor has been released.
 902  *
 903  * @param RunRef
 904  *        Pointer to a rundown reference descriptor.
 905  *
 906  * @return None.
 907  *
 908  * @remarks This is the internal macro for system use only. If a wait is actually
 909  *          necessary, then the slow path is taken through the exported function.
 910  *
 911  *--*/
 912 FORCEINLINE
 913 VOID
 914 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 915 {
 916     ULONG_PTR Value;
 917
 918     /* Set the active bit */
 919     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 920     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 921     {
 922         /* If the the rundown wasn't already active, then take the long path */
 923         ExfWaitForRundownProtectionRelease(RunRef);
 924     }
 925 }
 926
 927 /*++
 928  * @name ExRundownCompleted
 929  * INTERNAL MACRO
 930  *
 931  *     The ExRundownCompleted routine completes the rundown of the specified
 932  *     descriptor by setting the active bit.
 933  *
 934  * @param RunRef
 935  *        Pointer to a rundown reference descriptor.
 936  *
 937  * @return None.
 938  *
 939  * @remarks This is the internal macro for system use only.
 940  *
 941  *--*/
 942 FORCEINLINE
 943 VOID
 944 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 945 {
 946     /* Sanity check */
 947     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 948
 949     /* Mark the counter as active */
 950     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 951 }
 952
 953 /* PUSHLOCKS *****************************************************************/
 954
 955 /* FIXME: VERIFY THESE! */
 956
 957 VOID
 958 FASTCALL
 959 ExBlockPushLock(
 960     IN PEX_PUSH_LOCK PushLock,
 961     IN PVOID WaitBlock
 962 );
 963
 964 VOID
 965 FASTCALL
 966 ExfUnblockPushLock(
 967     IN PEX_PUSH_LOCK PushLock,
 968     IN PVOID CurrentWaitBlock
 969 );
 970
 971 VOID
 972 FASTCALL
 973 ExWaitForUnblockPushLock(
 974     IN PEX_PUSH_LOCK PushLock,
 975     IN PVOID WaitBlock
 976 );
 977
 978 /*++
 979  * @name _ExInitializePushLock
 980  * INTERNAL MACRO
 981  *
 982  *     The _ExInitializePushLock macro initializes a PushLock.
 983  *
 984  * @params PushLock
 985  *         Pointer to the pushlock which is to be initialized.
 986  *
 987  * @return None.
 988  *
 989  * @remarks None.
 990  *
 991  *--*/
 992 FORCEINLINE
 993 VOID
 994 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 995 {
 996     /* Set the value to 0 */
 997     PushLock->Ptr = 0;
 998 }
 999 #define ExInitializePushLock _ExInitializePushLock
1000
1001 /*++
1002  * @name ExAcquirePushLockExclusive
1003  * INTERNAL MACRO
1004  *
1005  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1006  *
1007  * @params PushLock
1008  *         Pointer to the pushlock which is to be acquired.
1009  *
1010  * @return None.
1011  *
1012  * @remarks The function attempts the quickest route to acquire the lock, which is
1013  *          to simply set the lock bit.
1014  *          However, if the pushlock is already shared, the slower path is taken.
1015  *
1016  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1017  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1018  *
1019  *--*/
1020 FORCEINLINE
1021 VOID
1022 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1023 {
1024     /* Try acquiring the lock */
1025     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1026     {
1027         /* Someone changed it, use the slow path */
1028         ExfAcquirePushLockExclusive(PushLock);
1029     }
1030
1031     /* Sanity check */
1032     ASSERT(PushLock->Locked);
1033 }
1034
1035 /*++
1036 * @name ExTryToAcquirePushLockExclusive
1037 * INTERNAL MACRO
1038 *
1039 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1040 *
1041 * @params PushLock
1042 *         Pointer to the pushlock which is to be acquired.
1043 *
1044 * @return None.
1045 *
1046 * @remarks The function attempts the quickest route to acquire the lock, which is
1047 *          to simply set the lock bit.
1048 *          However, if the pushlock is already shared, the slower path is taken.
1049 *
1050 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1051 *          This macro should usually be paired up with KeAcquireCriticalRegion.
1052 *
1053 *--*/
1054 FORCEINLINE
1055 BOOLEAN
1056 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1057 {
1058     /* Try acquiring the lock */
1059     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1060     {
1061         /* Can't acquire */
1062         return FALSE;
1063     }
1064
1065     /* Got acquired */
1066     ASSERT (PushLock->Locked);
1067     return TRUE;
1068 }
1069
1070 /*++
1071  * @name ExAcquirePushLockShared
1072  * INTERNAL MACRO
1073  *
1074  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1075  *
1076  * @params PushLock
1077  *         Pointer to the pushlock which is to be acquired.
1078  *
1079  * @return None.
1080  *
1081  * @remarks The function attempts the quickest route to acquire the lock, which is
1082  *          to simply set the lock bit and set the share count to one.
1083  *          However, if the pushlock is already shared, the slower path is taken.
1084  *
1085  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1086  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1087  *
1088  *--*/
1089 FORCEINLINE
1090 VOID
1091 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1092 {
1093     EX_PUSH_LOCK NewValue;
1094
1095     /* Try acquiring the lock */
1096     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1097     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1098     {
1099         /* Someone changed it, use the slow path */
1100         ExfAcquirePushLockShared(PushLock);
1101     }
1102
1103     /* Sanity checks */
1104     ASSERT(PushLock->Locked);
1105 }
1106
1107 /*++
1108  * @name ExConvertPushLockSharedToExclusive
1109  * INTERNAL MACRO
1110  *
1111  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1112  *     pushlock to a shared pushlock.
1113  *
1114  * @params PushLock
1115  *         Pointer to the pushlock which is to be converted.
1116  *
1117  * @return FALSE if conversion failed, TRUE otherwise.
1118  *
1119  * @remarks The function attempts the quickest route to convert the lock, which is
1120  *          to simply set the lock bit and remove any other bits.
1121  *
1122  *--*/
1123 FORCEINLINE
1124 BOOLEAN
1125 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1126 {
1127     EX_PUSH_LOCK OldValue;
1128
1129     /* Set the expected old value */
1130     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1131
1132     /* Try converting the lock */
1133     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1134         OldValue.Ptr)
1135     {
1136         /* Conversion failed */
1137         return FALSE;
1138     }
1139
1140     /* Sanity check */
1141     ASSERT(PushLock->Locked);
1142     return TRUE;
1143 }
1144
1145 /*++
1146  * @name ExWaitOnPushLock
1147  * INTERNAL MACRO
1148  *
1149  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1150  *
1151  * @params PushLock
1152  *         Pointer to a pushlock.
1153  *
1154  * @return None.
1155  *
1156  * @remarks The function attempts to get any exclusive waiters out of their slow
1157  *          path by forcing an instant acquire/release operation.
1158  *
1159  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1160  *
1161  *--*/
1162 FORCEINLINE
1163 VOID
1164 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1165 {
1166     /* Check if we're locked */
1167     if (PushLock->Locked)
1168     {
1169         /* Acquire the lock */
1170         ExfAcquirePushLockExclusive(PushLock);
1171         ASSERT(PushLock->Locked);
1172
1173         /* Release it */
1174         ExfReleasePushLockExclusive(PushLock);
1175     }
1176 }
1177
1178 /*++
1179  * @name ExReleasePushLockShared
1180  * INTERNAL MACRO
1181  *
1182  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1183  *
1184  * @params PushLock
1185  *         Pointer to a previously acquired pushlock.
1186  *
1187  * @return None.
1188  *
1189  * @remarks The function attempts the quickest route to release the lock, which is
1190  *          to simply decrease the share count and remove the lock bit.
1191  *          However, if the pushlock is being waited on then the long path is taken.
1192  *
1193  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1194  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1195  *
1196  *--*/
1197 FORCEINLINE
1198 VOID
1199 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1200 {
1201     EX_PUSH_LOCK OldValue;
1202
1203     /* Sanity checks */
1204     ASSERT(PushLock->Locked);
1205
1206     /* Try to clear the pushlock */
1207     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1208     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1209     {
1210         /* There are still other people waiting on it */
1211         ExfReleasePushLockShared(PushLock);
1212     }
1213 }
1214
1215 /*++
1216  * @name ExReleasePushLockExclusive
1217  * INTERNAL MACRO
1218  *
1219  *     The ExReleasePushLockExclusive macro releases a previously
1220  *     exclusively acquired PushLock.
1221  *
1222  * @params PushLock
1223  *         Pointer to a previously acquired pushlock.
1224  *
1225  * @return None.
1226  *
1227  * @remarks The function attempts the quickest route to release the lock, which is
1228  *          to simply clear the locked bit.
1229  *          However, if the pushlock is being waited on, the slow path is taken
1230  *          in an attempt to wake up the lock.
1231  *
1232  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1233  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1234  *
1235  *--*/
1236 FORCEINLINE
1237 VOID
1238 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1239 {
1240     EX_PUSH_LOCK OldValue;
1241
1242     /* Sanity checks */
1243     ASSERT(PushLock->Locked);
1244
1245     /* Unlock the pushlock */
1246     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1247                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1248
1249     /* Sanity checks */
1250     ASSERT(OldValue.Locked);
1251     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1252
1253     /* Check if anyone is waiting on it and it's not already waking*/
1254     if ((OldValue.Waiting) && !(OldValue.Waking))
1255     {
1256         /* Wake it up */
1257         ExfTryToWakePushLock(PushLock);
1258     }
1259 }
1260
1261 /*++
1262  * @name ExReleasePushLock
1263  * INTERNAL MACRO
1264  *
1265  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1266  *
1267  * @params PushLock
1268  *         Pointer to a previously acquired pushlock.
1269  *
1270  * @return None.
1271  *
1272  * @remarks The function attempts the quickest route to release the lock, which is
1273  *          to simply clear all the fields and decrease the share count if required.
1274  *          However, if the pushlock is being waited on then the long path is taken.
1275  *
1276  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1277  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1278  *
1279  *--*/
1280 FORCEINLINE
1281 VOID
1282 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1283 {
1284     EX_PUSH_LOCK OldValue = *PushLock;
1285     EX_PUSH_LOCK NewValue;
1286
1287     /* Sanity checks */
1288     ASSERT(OldValue.Locked);
1289
1290     /* Check if the pushlock is shared */
1291     if (OldValue.Shared > 1)
1292     {
1293         /* Decrease the share count */
1294         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1295     }
1296     else
1297     {
1298         /* Clear the pushlock entirely */
1299         NewValue.Value = 0;
1300     }
1301
1302     /* Check if nobody is waiting on us and try clearing the lock here */
1303     if ((OldValue.Waiting) ||
1304         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1305          OldValue.Ptr))
1306     {
1307         /* We have waiters, use the long path */
1308         ExfReleasePushLock(PushLock);
1309     }
1310 }
1311
1312 /* FAST MUTEX INLINES *********************************************************/
1313
1314 FORCEINLINE
1315 VOID
1316 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1317 {
1318     PKTHREAD Thread = KeGetCurrentThread();
1319
1320     /* Sanity check */
1321     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1322            (Thread->CombinedApcDisable != 0) ||
1323            (Thread->Teb == NULL) ||
1324            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1325     ASSERT(FastMutex->Owner != Thread);
1326
1327     /* Decrease the count */
1328     if (InterlockedDecrement(&FastMutex->Count))
1329     {
1330         /* Someone is still holding it, use slow path */
1331         KiAcquireFastMutex(FastMutex);
1332     }
1333
1334     /* Set the owner */
1335     FastMutex->Owner = Thread;
1336 }
1337
1338 FORCEINLINE
1339 VOID
1340 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1341 {
1342     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1343            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1344            (KeGetCurrentThread()->Teb == NULL) ||
1345            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1346     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1347
1348     /* Erase the owner */
1349     FastMutex->Owner = NULL;
1350
1351     /* Increase the count */
1352     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1353     {
1354         /* Someone was waiting for it, signal the waiter */
1355         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1356     }
1357 }
1358
1359 FORCEINLINE
1360 VOID
1361 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1362 {
1363     KIRQL OldIrql;
1364     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1365
1366     /* Raise IRQL to APC */
1367     KeRaiseIrql(APC_LEVEL, &OldIrql);
1368
1369     /* Decrease the count */
1370     if (InterlockedDecrement(&FastMutex->Count))
1371     {
1372         /* Someone is still holding it, use slow path */
1373         KiAcquireFastMutex(FastMutex);
1374     }
1375
1376     /* Set the owner and IRQL */
1377     FastMutex->Owner = KeGetCurrentThread();
1378     FastMutex->OldIrql = OldIrql;
1379 }
1380
1381 FORCEINLINE
1382 VOID
1383 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1384 {
1385     KIRQL OldIrql;
1386     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1387
1388     /* Erase the owner */
1389     FastMutex->Owner = NULL;
1390     OldIrql = (KIRQL)FastMutex->OldIrql;
1391
1392     /* Increase the count */
1393     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1394     {
1395         /* Someone was waiting for it, signal the waiter */
1396         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1397     }
1398
1399     /* Lower IRQL back */
1400     KeLowerIrql(OldIrql);
1401 }
1402
1403 FORCEINLINE
1404 BOOLEAN
1405 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1406 {
1407     KIRQL OldIrql;
1408     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1409
1410     /* Raise to APC_LEVEL */
1411     KeRaiseIrql(APC_LEVEL, &OldIrql);
1412
1413     /* Check if we can quickly acquire it */
1414     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1415     {
1416         /* We have, set us as owners */
1417         FastMutex->Owner = KeGetCurrentThread();
1418         FastMutex->OldIrql = OldIrql;
1419         return TRUE;
1420     }
1421     else
1422     {
1423         /* Acquire attempt failed */
1424         KeLowerIrql(OldIrql);
1425         YieldProcessor();
1426         return FALSE;
1427     }
1428 }
1429
1430 FORCEINLINE
1431 VOID
1432 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1433 {
1434     /* Enter the Critical Region */
1435     KeEnterCriticalRegion();
1436
1437     /* Acquire the mutex unsafely */
1438     _ExAcquireFastMutexUnsafe(FastMutex);
1439 }
1440
1441 FORCEINLINE
1442 VOID
1443 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1444 {
1445     /* Release the mutex unsafely */
1446     _ExReleaseFastMutexUnsafe(FastMutex);
1447
1448     /* Leave the critical region */
1449     KeLeaveCriticalRegion();
1450 }
1451
1452 /* OTHER FUNCTIONS **********************************************************/
1453
1454 BOOLEAN
1455 NTAPI
1456 ExTryToAcquireResourceExclusiveLite(
1457     IN PERESOURCE Resource
1458 );
1459
1460 NTSTATUS
1461 ExpSetTimeZoneInformation(
1462     IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1463 );
1464
1465 BOOLEAN
1466 NTAPI
1467 ExAcquireTimeRefreshLock(
1468     IN BOOLEAN Wait
1469 );
1470
1471 VOID
1472 NTAPI
1473 ExReleaseTimeRefreshLock(
1474     VOID
1475 );
1476
1477 VOID
1478 NTAPI
1479 ExUpdateSystemTimeFromCmos(
1480     IN BOOLEAN UpdateInterruptTime,
1481     IN ULONG MaxSepInSeconds
1482 );
1483
1484 VOID
1485 NTAPI
1486 ExAllocateLocallyUniqueId(
1487     OUT LUID *LocallyUniqueId
1488 );
1489
1490 VOID
1491 NTAPI
1492 ExTimerRundown(
1493     VOID
1494 );
1495
1496 INIT_FUNCTION
1497 VOID
1498 NTAPI
1499 HeadlessInit(
1500     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1501 );
1502
1503 INIT_FUNCTION
1504 VOID
1505 NTAPI
1506 XIPInit(
1507     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1508 );
1509
1510 #define InterlockedDecrementUL(Addend) \
1511    (ULONG)InterlockedDecrement((PLONG)(Addend))
1512
1513 #define InterlockedIncrementUL(Addend) \
1514    (ULONG)InterlockedIncrement((PLONG)(Addend))
1515
1516 #define InterlockedExchangeUL(Target, Value) \
1517    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1518
1519 #define InterlockedExchangeAddUL(Addend, Value) \
1520    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1521
1522 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1523    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1524
1525 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1526    (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1527
1528 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1529    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))