ntoskrnl/include/internal/ex.h

   1 #pragma once
   2
   3 /* GLOBAL VARIABLES *********************************************************/
   4
   5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   6 extern LARGE_INTEGER ExpTimeZoneBias;
   7 extern ULONG ExpTimeZoneId;
   8 extern ULONG ExpTickCountMultiplier;
   9 extern ULONG ExpLastTimeZoneBias;
  10 extern POBJECT_TYPE ExEventPairObjectType;
  11 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  12 extern FAST_MUTEX ExpEnvironmentLock;
  13 extern ERESOURCE ExpFirmwareTableResource;
  14 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  15 extern BOOLEAN ExpIsWinPEMode;
  16 extern LIST_ENTRY ExpSystemResourcesList;
  17 extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  18 extern ULONG ExpUnicodeCaseTableDataOffset;
  19 extern PVOID ExpNlsSectionPointer;
  20 extern ULONG NtGlobalFlag;
  21 extern UNICODE_STRING NtSystemRoot;
  22 extern ULONG ExpInitializationPhase;
  23 extern ULONG ExpAltTimeZoneBias;
  24 extern LIST_ENTRY ExSystemLookasideListHead;
  25 extern PCALLBACK_OBJECT PowerStateCallback;
  26 extern LIST_ENTRY ExPoolLookasideListHead;
  27 extern LIST_ENTRY ExpNonPagedLookasideListHead;
  28 extern LIST_ENTRY ExpPagedLookasideListHead;
  29 extern KSPIN_LOCK ExpNonPagedLookasideListLock;
  30 extern KSPIN_LOCK ExpPagedLookasideListLock;
  31
  32 /*
  33  * NT/Cm Version Info variables
  34  */
  35 extern ULONG NtMajorVersion;
  36 extern ULONG NtMinorVersion;
  37 extern ULONG NtBuildNumber;
  38 extern ULONG CmNtSpBuildNumber;
  39 extern ULONG CmNtCSDVersion;
  40 extern ULONG CmNtCSDReleaseType;
  41 extern UNICODE_STRING CmVersionString;
  42 extern UNICODE_STRING CmCSDVersionString;
  43 extern CHAR NtBuildLab[];
  44
  45 // #ifdef _WINKD_
  46 /*
  47  * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
  48  */
  49 typedef enum _WINKD_WORKER_STATE
  50 {
  51     WinKdWorkerReady = 0,
  52     WinKdWorkerStart,
  53     WinKdWorkerInitialized
  54 } WINKD_WORKER_STATE;
  55
  56 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem;
  57 extern WINKD_WORKER_STATE ExpDebuggerWork;
  58 extern PEPROCESS ExpDebuggerProcessAttach;
  59 extern PEPROCESS ExpDebuggerProcessKill;
  60 extern ULONG_PTR ExpDebuggerPageIn;
  61 VOID NTAPI ExpDebuggerWorker(IN PVOID Context);
  62 // #endif /* _WINKD_ */
  63
  64 #ifdef _WIN64
  65 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
  66 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
  67 #else
  68 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
  69 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
  70 #endif
  71 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - 31)
  72
  73 typedef union _EXHANDLE
  74 {
  75      struct
  76      {
  77          ULONG_PTR TagBits:2;
  78          ULONG_PTR Index:29;
  79      };
  80      struct
  81      {
  82          ULONG_PTR TagBits2:2;
  83          ULONG_PTR LowIndex:HANDLE_LOW_BITS;
  84          ULONG_PTR MidIndex:HANDLE_HIGH_BITS;
  85          ULONG_PTR HighIndex:HANDLE_HIGH_BITS;
  86          ULONG_PTR KernelFlag:KERNEL_FLAG_BITS;
  87      };
  88      HANDLE GenericHandleOverlay;
  89      ULONG_PTR Value;
  90 } EXHANDLE, *PEXHANDLE;
  91
  92 typedef struct _ETIMER
  93 {
  94     KTIMER KeTimer;
  95     KAPC TimerApc;
  96     KDPC TimerDpc;
  97     LIST_ENTRY ActiveTimerListEntry;
  98     KSPIN_LOCK Lock;
  99     LONG Period;
 100     BOOLEAN ApcAssociated;
 101     BOOLEAN WakeTimer;
 102     LIST_ENTRY WakeTimerListEntry;
 103 } ETIMER, *PETIMER;
 104
 105 typedef struct
 106 {
 107     PCALLBACK_OBJECT *CallbackObject;
 108     PWSTR Name;
 109 } SYSTEM_CALLBACKS;
 110
 111 typedef struct _HARDERROR_USER_PARAMETERS
 112 {
 113     ULONG_PTR Parameters[MAXIMUM_HARDERROR_PARAMETERS];
 114     UNICODE_STRING Strings[MAXIMUM_HARDERROR_PARAMETERS];
 115     WCHAR Buffer[ANYSIZE_ARRAY];
 116 } HARDERROR_USER_PARAMETERS, *PHARDERROR_USER_PARAMETERS;
 117
 118 #define MAX_FAST_REFS           7
 119
 120 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
 121 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
 122 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
 123 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
 124 #define ExRundownCompleted                              _ExRundownCompleted
 125 #define ExGetPreviousMode                               KeGetPreviousMode
 126
 127
 128 //
 129 // Various bits tagged on the handle or handle table
 130 //
 131 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
 132 #define FREE_HANDLE_MASK                -1
 133
 134 //
 135 // Number of entries in each table level
 136 //
 137 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
 138 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
 139 #define HIGH_LEVEL_ENTRIES  (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
 140
 141 //
 142 // Maximum index in each table level before we need another table
 143 //
 144 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
 145 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 146 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
 147
 148 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
 149 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
 150 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
 151
 152 NTSTATUS
 153 NTAPI
 154 ExGetPoolTagInfo(
 155     IN PSYSTEM_POOLTAG_INFORMATION SystemInformation,
 156     IN ULONG SystemInformationLength,
 157     IN OUT PULONG ReturnLength OPTIONAL
 158 );
 159
 160 /* INITIALIZATION FUNCTIONS *************************************************/
 161
 162 BOOLEAN
 163 NTAPI
 164 ExpWin32kInit(VOID);
 165
 166 VOID
 167 NTAPI
 168 ExInit2(VOID);
 169
 170 VOID
 171 NTAPI
 172 Phase1Initialization(
 173     IN PVOID Context
 174 );
 175
 176 VOID
 177 NTAPI
 178 ExpInitializePushLocks(VOID);
 179
 180 BOOLEAN
 181 NTAPI
 182 ExRefreshTimeZoneInformation(
 183     IN PLARGE_INTEGER SystemBootTime
 184 );
 185
 186 VOID
 187 NTAPI
 188 ExpInitializeWorkerThreads(VOID);
 189
 190 VOID
 191 NTAPI
 192 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
 193
 194 VOID
 195 NTAPI
 196 ExpInitLookasideLists(VOID);
 197
 198 VOID
 199 NTAPI
 200 ExInitializeSystemLookasideList(
 201     IN PGENERAL_LOOKASIDE List,
 202     IN POOL_TYPE Type,
 203     IN ULONG Size,
 204     IN ULONG Tag,
 205     IN USHORT MaximumDepth,
 206     IN PLIST_ENTRY ListHead
 207 );
 208
 209 BOOLEAN
 210 NTAPI
 211 ExpInitializeCallbacks(VOID);
 212
 213 VOID
 214 NTAPI
 215 ExpInitUuids(VOID);
 216
 217 VOID
 218 NTAPI
 219 ExpInitializeExecutive(
 220     IN ULONG Cpu,
 221     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 222 );
 223
 224 BOOLEAN
 225 NTAPI
 226 ExpInitializeEventImplementation(VOID);
 227
 228 BOOLEAN
 229 NTAPI
 230 ExpInitializeKeyedEventImplementation(VOID);
 231
 232 BOOLEAN
 233 NTAPI
 234 ExpInitializeEventPairImplementation(VOID);
 235
 236 BOOLEAN
 237 NTAPI
 238 ExpInitializeSemaphoreImplementation(VOID);
 239
 240 BOOLEAN
 241 NTAPI
 242 ExpInitializeMutantImplementation(VOID);
 243
 244 BOOLEAN
 245 NTAPI
 246 ExpInitializeTimerImplementation(VOID);
 247
 248 BOOLEAN
 249 NTAPI
 250 ExpInitializeProfileImplementation(VOID);
 251
 252 VOID
 253 NTAPI
 254 ExpResourceInitialization(VOID);
 255
 256 VOID
 257 NTAPI
 258 ExInitPoolLookasidePointers(VOID);
 259
 260 /* Callback Functions ********************************************************/
 261
 262 VOID
 263 NTAPI
 264 ExInitializeCallBack(
 265     IN OUT PEX_CALLBACK Callback
 266 );
 267
 268 PEX_CALLBACK_ROUTINE_BLOCK
 269 NTAPI
 270 ExAllocateCallBack(
 271     IN PEX_CALLBACK_FUNCTION Function,
 272     IN PVOID Context
 273 );
 274
 275 VOID
 276 NTAPI
 277 ExFreeCallBack(
 278     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 279 );
 280
 281 BOOLEAN
 282 NTAPI
 283 ExCompareExchangeCallBack (
 284     IN OUT PEX_CALLBACK CallBack,
 285     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 286     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 287 );
 288
 289 PEX_CALLBACK_ROUTINE_BLOCK
 290 NTAPI
 291 ExReferenceCallBackBlock(
 292     IN OUT PEX_CALLBACK CallBack
 293 );
 294
 295 VOID
 296 NTAPI
 297 ExDereferenceCallBackBlock(
 298     IN OUT PEX_CALLBACK CallBack,
 299     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 300 );
 301
 302 PEX_CALLBACK_FUNCTION
 303 NTAPI
 304 ExGetCallBackBlockRoutine(
 305     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 306 );
 307
 308 PVOID
 309 NTAPI
 310 ExGetCallBackBlockContext(
 311     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 312 );
 313
 314 VOID
 315 NTAPI
 316 ExWaitForCallBacks(
 317     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 318 );
 319
 320 /* Rundown Functions ********************************************************/
 321
 322 VOID
 323 FASTCALL
 324 ExfInitializeRundownProtection(
 325      OUT PEX_RUNDOWN_REF RunRef
 326 );
 327
 328 VOID
 329 FASTCALL
 330 ExfReInitializeRundownProtection(
 331      OUT PEX_RUNDOWN_REF RunRef
 332 );
 333
 334 BOOLEAN
 335 FASTCALL
 336 ExfAcquireRundownProtection(
 337      IN OUT PEX_RUNDOWN_REF RunRef
 338 );
 339
 340 BOOLEAN
 341 FASTCALL
 342 ExfAcquireRundownProtectionEx(
 343      IN OUT PEX_RUNDOWN_REF RunRef,
 344      IN ULONG Count
 345 );
 346
 347 VOID
 348 FASTCALL
 349 ExfReleaseRundownProtection(
 350      IN OUT PEX_RUNDOWN_REF RunRef
 351 );
 352
 353 VOID
 354 FASTCALL
 355 ExfReleaseRundownProtectionEx(
 356      IN OUT PEX_RUNDOWN_REF RunRef,
 357      IN ULONG Count
 358 );
 359
 360 VOID
 361 FASTCALL
 362 ExfRundownCompleted(
 363      OUT PEX_RUNDOWN_REF RunRef
 364 );
 365
 366 VOID
 367 FASTCALL
 368 ExfWaitForRundownProtectionRelease(
 369      IN OUT PEX_RUNDOWN_REF RunRef
 370 );
 371
 372 /* HANDLE TABLE FUNCTIONS ***************************************************/
 373
 374 typedef BOOLEAN
 375 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 376     PHANDLE_TABLE_ENTRY HandleTableEntry,
 377     HANDLE Handle,
 378     PVOID Context
 379 );
 380
 381 typedef BOOLEAN
 382 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 383     IN PEPROCESS Process,
 384     IN PHANDLE_TABLE HandleTable,
 385     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 386     IN PHANDLE_TABLE_ENTRY NewEntry
 387 );
 388
 389 typedef BOOLEAN
 390 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 391     PHANDLE_TABLE_ENTRY HandleTableEntry,
 392     ULONG_PTR Context
 393 );
 394
 395 VOID
 396 NTAPI
 397 ExpInitializeHandleTables(
 398     VOID
 399 );
 400
 401 PHANDLE_TABLE
 402 NTAPI
 403 ExCreateHandleTable(
 404     IN PEPROCESS Process OPTIONAL
 405 );
 406
 407 VOID
 408 NTAPI
 409 ExUnlockHandleTableEntry(
 410     IN PHANDLE_TABLE HandleTable,
 411     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 412 );
 413
 414 HANDLE
 415 NTAPI
 416 ExCreateHandle(
 417     IN PHANDLE_TABLE HandleTable,
 418     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 419 );
 420
 421 VOID
 422 NTAPI
 423 ExDestroyHandleTable(
 424     IN PHANDLE_TABLE HandleTable,
 425     IN PVOID DestroyHandleProcedure OPTIONAL
 426 );
 427
 428 BOOLEAN
 429 NTAPI
 430 ExDestroyHandle(
 431     IN PHANDLE_TABLE HandleTable,
 432     IN HANDLE Handle,
 433     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 434 );
 435
 436 PHANDLE_TABLE_ENTRY
 437 NTAPI
 438 ExMapHandleToPointer(
 439     IN PHANDLE_TABLE HandleTable,
 440     IN HANDLE Handle
 441 );
 442
 443 PHANDLE_TABLE
 444 NTAPI
 445 ExDupHandleTable(
 446     IN PEPROCESS Process,
 447     IN PHANDLE_TABLE HandleTable,
 448     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 449     IN ULONG_PTR Mask
 450 );
 451
 452 BOOLEAN
 453 NTAPI
 454 ExChangeHandle(
 455     IN PHANDLE_TABLE HandleTable,
 456     IN HANDLE Handle,
 457     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 458     IN ULONG_PTR Context
 459 );
 460
 461 VOID
 462 NTAPI
 463 ExSweepHandleTable(
 464     IN PHANDLE_TABLE HandleTable,
 465     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 466     IN PVOID Context
 467 );
 468
 469 PHANDLE_TABLE_ENTRY
 470 NTAPI
 471 ExpLookupHandleTableEntry(
 472     IN PHANDLE_TABLE HandleTable,
 473     IN EXHANDLE Handle
 474 );
 475
 476 BOOLEAN
 477 NTAPI
 478 ExpLockHandleTableEntry(
 479     IN PHANDLE_TABLE HandleTable,
 480     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 481 );
 482
 483 /* PSEH EXCEPTION HANDLING **************************************************/
 484
 485 LONG
 486 NTAPI
 487 ExSystemExceptionFilter(VOID);
 488
 489 /* CALLBACKS *****************************************************************/
 490
 491 FORCEINLINE
 492 VOID
 493 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 494              IN PVOID Context,
 495              IN PVOID Argument1,
 496              IN PVOID Argument2)
 497 {
 498     PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 499     PEX_CALLBACK_FUNCTION Function;
 500
 501     /* Reference the block */
 502     CallbackBlock = ExReferenceCallBackBlock(Callback);
 503     if (CallbackBlock)
 504     {
 505         /* Get the function */
 506         Function = ExGetCallBackBlockRoutine(CallbackBlock);
 507
 508         /* Do the callback */
 509         Function(Context, Argument1, Argument2);
 510
 511         /* Now dereference it */
 512         ExDereferenceCallBackBlock(Callback, CallbackBlock);
 513     }
 514 }
 515
 516 /* FAST REFS ******************************************************************/
 517
 518 FORCEINLINE
 519 PVOID
 520 ExGetObjectFastReference(IN EX_FAST_REF FastRef)
 521 {
 522     /* Return the unbiased pointer */
 523     return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
 524 }
 525
 526 FORCEINLINE
 527 ULONG
 528 ExGetCountFastReference(IN EX_FAST_REF FastRef)
 529 {
 530     /* Return the reference count */
 531     return (ULONG)FastRef.RefCnt;
 532 }
 533
 534 FORCEINLINE
 535 VOID
 536 ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
 537                           IN OPTIONAL PVOID Object)
 538 {
 539     /* Sanity check */
 540     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 541
 542     /* Check if an object is being set */
 543     if (!Object)
 544     {
 545         /* Clear the field */
 546         FastRef->Object = NULL;
 547     }
 548     else
 549     {
 550         /* Otherwise, we assume the object was referenced and is ready */
 551         FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 552     }
 553 }
 554
 555 FORCEINLINE
 556 EX_FAST_REF
 557 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
 558 {
 559     EX_FAST_REF OldValue, NewValue;
 560
 561     /* Start reference loop */
 562     for (;;)
 563     {
 564         /* Get the current reference count */
 565         OldValue = *FastRef;
 566         if (OldValue.RefCnt)
 567         {
 568             /* Increase the reference count */
 569             NewValue.Value = OldValue.Value - 1;
 570             NewValue.Object = ExpChangePushlock(&FastRef->Object,
 571                                                 NewValue.Object,
 572                                                 OldValue.Object);
 573             if (NewValue.Object != OldValue.Object) continue;
 574         }
 575
 576         /* We are done */
 577         break;
 578     }
 579
 580     /* Return the old value */
 581     return OldValue;
 582 }
 583
 584 FORCEINLINE
 585 BOOLEAN
 586 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
 587                       IN PVOID Object)
 588 {
 589     EX_FAST_REF OldValue, NewValue;
 590
 591     /* Sanity checks */
 592     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 593
 594     /* Start update loop */
 595     for (;;)
 596     {
 597         /* Get the current reference count */
 598         OldValue = *FastRef;
 599
 600         /* Check if the current count is too high or if the pointer changed */
 601         if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
 602             ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
 603         {
 604             /* Fail */
 605             return FALSE;
 606         }
 607
 608         /* Update the reference count */
 609         NewValue.Value = OldValue.Value + MAX_FAST_REFS;
 610         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 611                                             NewValue.Object,
 612                                             OldValue.Object);
 613         if (NewValue.Object != OldValue.Object) continue;
 614
 615         /* We are done */
 616         break;
 617     }
 618
 619     /* Return success */
 620     return TRUE;
 621 }
 622
 623 FORCEINLINE
 624 BOOLEAN
 625 ExReleaseFastReference(IN PEX_FAST_REF FastRef,
 626                        IN PVOID Object)
 627 {
 628     EX_FAST_REF OldValue, NewValue;
 629
 630     /* Sanity checks */
 631     ASSERT(Object != NULL);
 632     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 633
 634     /* Start reference loop */
 635     for (;;)
 636     {
 637         /* Get the current reference count */
 638         OldValue = *FastRef;
 639
 640         /* Check if we're full if if the pointer changed */
 641         if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
 642
 643         /* Decrease the reference count */
 644         NewValue.Value = OldValue.Value + 1;
 645         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 646                                             NewValue.Object,
 647                                             OldValue.Object);
 648         if (NewValue.Object != OldValue.Object) continue;
 649
 650         /* We are done */
 651         break;
 652     }
 653
 654     /* Return success */
 655     return TRUE;
 656 }
 657
 658 FORCEINLINE
 659 EX_FAST_REF
 660 ExSwapFastReference(IN PEX_FAST_REF FastRef,
 661                     IN PVOID Object)
 662 {
 663     EX_FAST_REF NewValue, OldValue;
 664
 665     /* Sanity check */
 666     ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
 667
 668     /* Check if an object is being set */
 669     if (!Object)
 670     {
 671         /* Clear the field */
 672         NewValue.Object = NULL;
 673     }
 674     else
 675     {
 676         /* Otherwise, we assume the object was referenced and is ready */
 677         NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 678     }
 679
 680     /* Update the object */
 681     OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
 682     return OldValue;
 683 }
 684
 685 FORCEINLINE
 686 EX_FAST_REF
 687 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
 688                            IN PVOID Object,
 689                            IN PVOID OldObject)
 690 {
 691     EX_FAST_REF OldValue, NewValue;
 692
 693     /* Sanity check and start swap loop */
 694     ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
 695     for (;;)
 696     {
 697         /* Get the current value */
 698         OldValue = *FastRef;
 699
 700         /* Make sure there's enough references to swap */
 701         if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
 702
 703         /* Check if we have an object to swap */
 704         if (Object)
 705         {
 706             /* Set up the value with maximum fast references */
 707             NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
 708         }
 709         else
 710         {
 711             /* Write the object address itself (which is empty) */
 712             NewValue.Value = (ULONG_PTR)Object;
 713         }
 714
 715         /* Do the actual compare exchange */
 716         NewValue.Object = ExpChangePushlock(&FastRef->Object,
 717                                             NewValue.Object,
 718                                             OldValue.Object);
 719         if (NewValue.Object != OldValue.Object) continue;
 720
 721         /* All done */
 722         break;
 723     }
 724
 725     /* Return the old value */
 726     return OldValue;
 727 }
 728
 729 /* RUNDOWN *******************************************************************/
 730
 731 /*++
 732  * @name ExfAcquireRundownProtection
 733  * INTERNAL MACRO
 734  *
 735  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 736  *     the specified descriptor.
 737  *
 738  * @param RunRef
 739  *        Pointer to a rundown reference descriptor.
 740  *
 741  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 742  *
 743  * @remarks This is the internal macro for system use only.In case the rundown
 744  *          was active, then the slow-path will be called through the exported
 745  *          function.
 746  *
 747  *--*/
 748 FORCEINLINE
 749 BOOLEAN
 750 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 751 {
 752     ULONG_PTR Value, NewValue;
 753
 754     /* Get the current value and mask the active bit */
 755     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 756
 757     /* Add a reference */
 758     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 759
 760     /* Change the value */
 761     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 762     if (NewValue != Value)
 763     {
 764         /* Rundown was active, use long path */
 765         return ExfAcquireRundownProtection(RunRef);
 766     }
 767
 768     /* Success */
 769     return TRUE;
 770 }
 771
 772 /*++
 773  * @name ExReleaseRundownProtection
 774  * INTERNAL MACRO
 775  *
 776  *     The ExReleaseRundownProtection routine releases rundown protection for
 777  *     the specified descriptor.
 778  *
 779  * @param RunRef
 780  *        Pointer to a rundown reference descriptor.
 781  *
 782  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 783  *
 784  * @remarks This is the internal macro for system use only.In case the rundown
 785  *          was active, then the slow-path will be called through the exported
 786  *          function.
 787  *
 788  *--*/
 789 FORCEINLINE
 790 VOID
 791 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 792 {
 793     ULONG_PTR Value, NewValue;
 794
 795     /* Get the current value and mask the active bit */
 796     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 797
 798     /* Remove a reference */
 799     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 800
 801     /* Change the value */
 802     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 803
 804     /* Check if the rundown was active */
 805     if (NewValue != Value)
 806     {
 807         /* Rundown was active, use long path */
 808         ExfReleaseRundownProtection(RunRef);
 809     }
 810     else
 811     {
 812         /* Sanity check */
 813         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 814     }
 815 }
 816
 817 /*++
 818  * @name ExInitializeRundownProtection
 819  * INTERNAL MACRO
 820  *
 821  *     The ExInitializeRundownProtection routine initializes a rundown
 822  *     protection descriptor.
 823  *
 824  * @param RunRef
 825  *        Pointer to a rundown reference descriptor.
 826  *
 827  * @return None.
 828  *
 829  * @remarks This is the internal macro for system use only.
 830  *
 831  *--*/
 832 FORCEINLINE
 833 VOID
 834 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 835 {
 836     /* Set the count to zero */
 837     RunRef->Count = 0;
 838 }
 839
 840 /*++
 841  * @name ExWaitForRundownProtectionRelease
 842  * INTERNAL MACRO
 843  *
 844  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 845  *     rundown descriptor has been released.
 846  *
 847  * @param RunRef
 848  *        Pointer to a rundown reference descriptor.
 849  *
 850  * @return None.
 851  *
 852  * @remarks This is the internal macro for system use only. If a wait is actually
 853  *          necessary, then the slow path is taken through the exported function.
 854  *
 855  *--*/
 856 FORCEINLINE
 857 VOID
 858 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 859 {
 860     ULONG_PTR Value;
 861
 862     /* Set the active bit */
 863     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 864     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 865     {
 866         /* If the the rundown wasn't already active, then take the long path */
 867         ExfWaitForRundownProtectionRelease(RunRef);
 868     }
 869 }
 870
 871 /*++
 872  * @name ExRundownCompleted
 873  * INTERNAL MACRO
 874  *
 875  *     The ExRundownCompleted routine completes the rundown of the specified
 876  *     descriptor by setting the active bit.
 877  *
 878  * @param RunRef
 879  *        Pointer to a rundown reference descriptor.
 880  *
 881  * @return None.
 882  *
 883  * @remarks This is the internal macro for system use only.
 884  *
 885  *--*/
 886 FORCEINLINE
 887 VOID
 888 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 889 {
 890     /* Sanity check */
 891     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 892
 893     /* Mark the counter as active */
 894     ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
 895 }
 896
 897 /* PUSHLOCKS *****************************************************************/
 898
 899 /* FIXME: VERIFY THESE! */
 900
 901 VOID
 902 FASTCALL
 903 ExBlockPushLock(
 904     IN PEX_PUSH_LOCK PushLock,
 905     IN PVOID WaitBlock
 906 );
 907
 908 VOID
 909 FASTCALL
 910 ExfUnblockPushLock(
 911     IN PEX_PUSH_LOCK PushLock,
 912     IN PVOID CurrentWaitBlock
 913 );
 914
 915 VOID
 916 FASTCALL
 917 ExWaitForUnblockPushLock(
 918     IN PEX_PUSH_LOCK PushLock,
 919     IN PVOID WaitBlock
 920 );
 921
 922 /*++
 923  * @name _ExInitializePushLock
 924  * INTERNAL MACRO
 925  *
 926  *     The _ExInitializePushLock macro initializes a PushLock.
 927  *
 928  * @params PushLock
 929  *         Pointer to the pushlock which is to be initialized.
 930  *
 931  * @return None.
 932  *
 933  * @remarks None.
 934  *
 935  *--*/
 936 FORCEINLINE
 937 VOID
 938 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock)
 939 {
 940     /* Set the value to 0 */
 941     PushLock->Ptr = 0;
 942 }
 943 #define ExInitializePushLock _ExInitializePushLock
 944
 945 /*++
 946  * @name ExAcquirePushLockExclusive
 947  * INTERNAL MACRO
 948  *
 949  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 950  *
 951  * @params PushLock
 952  *         Pointer to the pushlock which is to be acquired.
 953  *
 954  * @return None.
 955  *
 956  * @remarks The function attempts the quickest route to acquire the lock, which is
 957  *          to simply set the lock bit.
 958  *          However, if the pushlock is already shared, the slower path is taken.
 959  *
 960  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 961  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 962  *
 963  *--*/
 964 FORCEINLINE
 965 VOID
 966 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 967 {
 968     /* Try acquiring the lock */
 969     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 970     {
 971         /* Someone changed it, use the slow path */
 972         ExfAcquirePushLockExclusive(PushLock);
 973     }
 974
 975     /* Sanity check */
 976     ASSERT(PushLock->Locked);
 977 }
 978
 979 /*++
 980 * @name ExTryToAcquirePushLockExclusive
 981 * INTERNAL MACRO
 982 *
 983 *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 984 *
 985 * @params PushLock
 986 *         Pointer to the pushlock which is to be acquired.
 987 *
 988 * @return None.
 989 *
 990 * @remarks The function attempts the quickest route to acquire the lock, which is
 991 *          to simply set the lock bit.
 992 *          However, if the pushlock is already shared, the slower path is taken.
 993 *
 994 *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 995 *          This macro should usually be paired up with KeAcquireCriticalRegion.
 996 *
 997 *--*/
 998 FORCEINLINE
 999 BOOLEAN
1000 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
1001 {
1002     /* Try acquiring the lock */
1003     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
1004     {
1005         /* Can't acquire */
1006         return FALSE;
1007     }
1008
1009     /* Got acquired */
1010     ASSERT (PushLock->Locked);
1011     return TRUE;
1012 }
1013
1014 /*++
1015  * @name ExAcquirePushLockShared
1016  * INTERNAL MACRO
1017  *
1018  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
1019  *
1020  * @params PushLock
1021  *         Pointer to the pushlock which is to be acquired.
1022  *
1023  * @return None.
1024  *
1025  * @remarks The function attempts the quickest route to acquire the lock, which is
1026  *          to simply set the lock bit and set the share count to one.
1027  *          However, if the pushlock is already shared, the slower path is taken.
1028  *
1029  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1030  *          This macro should usually be paired up with KeAcquireCriticalRegion.
1031  *
1032  *--*/
1033 FORCEINLINE
1034 VOID
1035 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
1036 {
1037     EX_PUSH_LOCK NewValue;
1038
1039     /* Try acquiring the lock */
1040     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1041     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
1042     {
1043         /* Someone changed it, use the slow path */
1044         ExfAcquirePushLockShared(PushLock);
1045     }
1046
1047     /* Sanity checks */
1048     ASSERT(PushLock->Locked);
1049 }
1050
1051 /*++
1052  * @name ExConvertPushLockSharedToExclusive
1053  * INTERNAL MACRO
1054  *
1055  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
1056  *     pushlock to a shared pushlock.
1057  *
1058  * @params PushLock
1059  *         Pointer to the pushlock which is to be converted.
1060  *
1061  * @return FALSE if conversion failed, TRUE otherwise.
1062  *
1063  * @remarks The function attempts the quickest route to convert the lock, which is
1064  *          to simply set the lock bit and remove any other bits.
1065  *
1066  *--*/
1067 FORCEINLINE
1068 BOOLEAN
1069 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
1070 {
1071     EX_PUSH_LOCK OldValue;
1072
1073     /* Set the expected old value */
1074     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1075
1076     /* Try converting the lock */
1077     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
1078         OldValue.Ptr)
1079     {
1080         /* Conversion failed */
1081         return FALSE;
1082     }
1083
1084     /* Sanity check */
1085     ASSERT(PushLock->Locked);
1086     return TRUE;
1087 }
1088
1089 /*++
1090  * @name ExWaitOnPushLock
1091  * INTERNAL MACRO
1092  *
1093  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1094  *
1095  * @params PushLock
1096  *         Pointer to a pushlock.
1097  *
1098  * @return None.
1099  *
1100  * @remarks The function attempts to get any exclusive waiters out of their slow
1101  *          path by forcing an instant acquire/release operation.
1102  *
1103  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1104  *
1105  *--*/
1106 FORCEINLINE
1107 VOID
1108 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
1109 {
1110     /* Check if we're locked */
1111     if (PushLock->Locked)
1112     {
1113         /* Acquire the lock */
1114         ExfAcquirePushLockExclusive(PushLock);
1115         ASSERT(PushLock->Locked);
1116
1117         /* Release it */
1118         ExfReleasePushLockExclusive(PushLock);
1119     }
1120 }
1121
1122 /*++
1123  * @name ExReleasePushLockShared
1124  * INTERNAL MACRO
1125  *
1126  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
1127  *
1128  * @params PushLock
1129  *         Pointer to a previously acquired pushlock.
1130  *
1131  * @return None.
1132  *
1133  * @remarks The function attempts the quickest route to release the lock, which is
1134  *          to simply decrease the share count and remove the lock bit.
1135  *          However, if the pushlock is being waited on then the long path is taken.
1136  *
1137  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1138  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1139  *
1140  *--*/
1141 FORCEINLINE
1142 VOID
1143 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
1144 {
1145     EX_PUSH_LOCK OldValue;
1146
1147     /* Sanity checks */
1148     ASSERT(PushLock->Locked);
1149
1150     /* Try to clear the pushlock */
1151     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
1152     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
1153     {
1154         /* There are still other people waiting on it */
1155         ExfReleasePushLockShared(PushLock);
1156     }
1157 }
1158
1159 /*++
1160  * @name ExReleasePushLockExclusive
1161  * INTERNAL MACRO
1162  *
1163  *     The ExReleasePushLockExclusive macro releases a previously
1164  *     exclusively acquired PushLock.
1165  *
1166  * @params PushLock
1167  *         Pointer to a previously acquired pushlock.
1168  *
1169  * @return None.
1170  *
1171  * @remarks The function attempts the quickest route to release the lock, which is
1172  *          to simply clear the locked bit.
1173  *          However, if the pushlock is being waited on, the slow path is taken
1174  *          in an attempt to wake up the lock.
1175  *
1176  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1177  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1178  *
1179  *--*/
1180 FORCEINLINE
1181 VOID
1182 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
1183 {
1184     EX_PUSH_LOCK OldValue;
1185
1186     /* Sanity checks */
1187     ASSERT(PushLock->Locked);
1188
1189     /* Unlock the pushlock */
1190     OldValue.Value = InterlockedExchangeAddSizeT((PSIZE_T)PushLock,
1191                                                  -(SSIZE_T)EX_PUSH_LOCK_LOCK);
1192
1193     /* Sanity checks */
1194     ASSERT(OldValue.Locked);
1195     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
1196
1197     /* Check if anyone is waiting on it and it's not already waking*/
1198     if ((OldValue.Waiting) && !(OldValue.Waking))
1199     {
1200         /* Wake it up */
1201         ExfTryToWakePushLock(PushLock);
1202     }
1203 }
1204
1205 /*++
1206  * @name ExReleasePushLock
1207  * INTERNAL MACRO
1208  *
1209  *     The ExReleasePushLock macro releases a previously acquired PushLock.
1210  *
1211  * @params PushLock
1212  *         Pointer to a previously acquired pushlock.
1213  *
1214  * @return None.
1215  *
1216  * @remarks The function attempts the quickest route to release the lock, which is
1217  *          to simply clear all the fields and decrease the share count if required.
1218  *          However, if the pushlock is being waited on then the long path is taken.
1219  *
1220  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1221  *          This macro should usually be paired up with KeLeaveCriticalRegion.
1222  *
1223  *--*/
1224 FORCEINLINE
1225 VOID
1226 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
1227 {
1228     EX_PUSH_LOCK OldValue = *PushLock;
1229     EX_PUSH_LOCK NewValue;
1230
1231     /* Sanity checks */
1232     ASSERT(OldValue.Locked);
1233
1234     /* Check if the pushlock is shared */
1235     if (OldValue.Shared > 1)
1236     {
1237         /* Decrease the share count */
1238         NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
1239     }
1240     else
1241     {
1242         /* Clear the pushlock entirely */
1243         NewValue.Value = 0;
1244     }
1245
1246     /* Check if nobody is waiting on us and try clearing the lock here */
1247     if ((OldValue.Waiting) ||
1248         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
1249          OldValue.Ptr))
1250     {
1251         /* We have waiters, use the long path */
1252         ExfReleasePushLock(PushLock);
1253     }
1254 }
1255
1256 /* FAST MUTEX INLINES *********************************************************/
1257
1258 FORCEINLINE
1259 VOID
1260 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
1261 {
1262     PKTHREAD Thread = KeGetCurrentThread();
1263
1264     /* Sanity check */
1265     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1266            (Thread->CombinedApcDisable != 0) ||
1267            (Thread->Teb == NULL) ||
1268            (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1269     ASSERT(FastMutex->Owner != Thread);
1270
1271     /* Decrease the count */
1272     if (InterlockedDecrement(&FastMutex->Count))
1273     {
1274         /* Someone is still holding it, use slow path */
1275         KiAcquireFastMutex(FastMutex);
1276     }
1277
1278     /* Set the owner */
1279     FastMutex->Owner = Thread;
1280 }
1281
1282 FORCEINLINE
1283 VOID
1284 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1285 {
1286     ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
1287            (KeGetCurrentThread()->CombinedApcDisable != 0) ||
1288            (KeGetCurrentThread()->Teb == NULL) ||
1289            (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
1290     ASSERT(FastMutex->Owner == KeGetCurrentThread());
1291
1292     /* Erase the owner */
1293     FastMutex->Owner = NULL;
1294
1295     /* Increase the count */
1296     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1297     {
1298         /* Someone was waiting for it, signal the waiter */
1299         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1300     }
1301 }
1302
1303 FORCEINLINE
1304 VOID
1305 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
1306 {
1307     KIRQL OldIrql;
1308     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1309
1310     /* Raise IRQL to APC */
1311     KeRaiseIrql(APC_LEVEL, &OldIrql);
1312
1313     /* Decrease the count */
1314     if (InterlockedDecrement(&FastMutex->Count))
1315     {
1316         /* Someone is still holding it, use slow path */
1317         KiAcquireFastMutex(FastMutex);
1318     }
1319
1320     /* Set the owner and IRQL */
1321     FastMutex->Owner = KeGetCurrentThread();
1322     FastMutex->OldIrql = OldIrql;
1323 }
1324
1325 FORCEINLINE
1326 VOID
1327 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
1328 {
1329     KIRQL OldIrql;
1330     ASSERT(KeGetCurrentIrql() == APC_LEVEL);
1331
1332     /* Erase the owner */
1333     FastMutex->Owner = NULL;
1334     OldIrql = (KIRQL)FastMutex->OldIrql;
1335
1336     /* Increase the count */
1337     if (InterlockedIncrement(&FastMutex->Count) <= 0)
1338     {
1339         /* Someone was waiting for it, signal the waiter */
1340         KeSetEventBoostPriority(&FastMutex->Event, NULL);
1341     }
1342
1343     /* Lower IRQL back */
1344     KeLowerIrql(OldIrql);
1345 }
1346
1347 FORCEINLINE
1348 BOOLEAN
1349 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
1350 {
1351     KIRQL OldIrql;
1352     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1353
1354     /* Raise to APC_LEVEL */
1355     KeRaiseIrql(APC_LEVEL, &OldIrql);
1356
1357     /* Check if we can quickly acquire it */
1358     if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
1359     {
1360         /* We have, set us as owners */
1361         FastMutex->Owner = KeGetCurrentThread();
1362         FastMutex->OldIrql = OldIrql;
1363         return TRUE;
1364     }
1365     else
1366     {
1367         /* Acquire attempt failed */
1368         KeLowerIrql(OldIrql);
1369         YieldProcessor();
1370         return FALSE;
1371     }
1372 }
1373
1374 FORCEINLINE
1375 VOID
1376 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
1377 {
1378     /* Enter the Critical Region */
1379     KeEnterCriticalRegion();
1380
1381     /* Acquire the mutex unsafely */
1382     _ExAcquireFastMutexUnsafe(FastMutex);
1383 }
1384
1385 FORCEINLINE
1386 VOID
1387 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
1388 {
1389     /* Release the mutex unsafely */
1390     _ExReleaseFastMutexUnsafe(FastMutex);
1391
1392     /* Leave the critical region */
1393     KeLeaveCriticalRegion();
1394 }
1395
1396 /* OTHER FUNCTIONS **********************************************************/
1397
1398 BOOLEAN
1399 NTAPI
1400 ExTryToAcquireResourceExclusiveLite(
1401     IN PERESOURCE Resource
1402 );
1403
1404 NTSTATUS
1405 ExpSetTimeZoneInformation(
1406     IN PTIME_ZONE_INFORMATION TimeZoneInformation
1407 );
1408
1409 BOOLEAN
1410 NTAPI
1411 ExAcquireTimeRefreshLock(
1412     IN BOOLEAN Wait
1413 );
1414
1415 VOID
1416 NTAPI
1417 ExReleaseTimeRefreshLock(
1418     VOID
1419 );
1420
1421 VOID
1422 NTAPI
1423 ExUpdateSystemTimeFromCmos(
1424     IN BOOLEAN UpdateInterruptTime,
1425     IN ULONG MaxSepInSeconds
1426 );
1427
1428 VOID
1429 NTAPI
1430 ExAllocateLocallyUniqueId(
1431     OUT LUID *LocallyUniqueId
1432 );
1433
1434 VOID
1435 NTAPI
1436 ExTimerRundown(
1437     VOID
1438 );
1439
1440 VOID
1441 NTAPI
1442 HeadlessInit(
1443     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1444 );
1445
1446 VOID
1447 NTAPI
1448 XIPInit(
1449     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1450 );
1451
1452 #define InterlockedDecrementUL(Addend) \
1453    (ULONG)InterlockedDecrement((PLONG)(Addend))
1454
1455 #define InterlockedIncrementUL(Addend) \
1456    (ULONG)InterlockedIncrement((PLONG)(Addend))
1457
1458 #define InterlockedExchangeUL(Target, Value) \
1459    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1460
1461 #define InterlockedExchangeAddUL(Addend, Value) \
1462    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1463
1464 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1465    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1466
1467 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1468    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))