reactos/ntoskrnl/include/internal/ex.h

   1 #ifndef __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
   2 #define __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
   3
   4 /* GLOBAL VARIABLES *********************************************************/
   5
   6 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo;
   7 extern LARGE_INTEGER ExpTimeZoneBias;
   8 extern ULONG ExpTimeZoneId;
   9 extern ULONG ExpTickCountMultiplier;
  10 extern ULONG ExpLastTimeZoneBias;
  11 extern POBJECT_TYPE ExEventPairObjectType;
  12 extern POBJECT_TYPE _ExEventObjectType, _ExSemaphoreObjectType;
  13 extern ULONG NtBuildNumber;
  14 extern ULONG NtMajorVersion;
  15 extern ULONG NtMinorVersion;
  16 extern FAST_MUTEX ExpEnvironmentLock;
  17 extern ERESOURCE ExpFirmwareTableResource;
  18 extern LIST_ENTRY ExpFirmwareTableProviderListHead;
  19 extern BOOLEAN ExpIsWinPEMode;
  20 ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
  21 ULONG ExpUnicodeCaseTableDataOffset;
  22 PVOID ExpNlsSectionPointer;
  23
  24 typedef struct _EXHANDLE
  25 {
  26     union
  27     {
  28         struct
  29         {
  30             ULONG TagBits:2;
  31             ULONG Index:30;
  32         };
  33         HANDLE GenericHandleOverlay;
  34         ULONG_PTR Value;
  35     };
  36 } EXHANDLE, *PEXHANDLE;
  37
  38 typedef struct _ETIMER
  39 {
  40     KTIMER KeTimer;
  41     KAPC TimerApc;
  42     KDPC TimerDpc;
  43     LIST_ENTRY ActiveTimerListEntry;
  44     KSPIN_LOCK Lock;
  45     LONG Period;
  46     BOOLEAN ApcAssociated;
  47     BOOLEAN WakeTimer;
  48     LIST_ENTRY WakeTimerListEntry;
  49 } ETIMER, *PETIMER;
  50
  51 typedef struct
  52 {
  53     PCALLBACK_OBJECT *CallbackObject;
  54     PWSTR Name;
  55 } SYSTEM_CALLBACKS;
  56
  57 #define MAX_FAST_REFS           7
  58
  59 /* Note: we only use a spinlock on SMP. On UP, we cli/sti intead */
  60 #ifndef CONFIG_SMP
  61 #define ExAcquireResourceLock(l, i) { \
  62     (void)i; \
  63     _disable(); \
  64 }
  65 #define ExReleaseResourceLock(l, i) _enable();
  66 #else
  67 #define ExAcquireResourceLock(l, i) KeAcquireSpinLock(l, i);
  68 #define ExReleaseResourceLock(l, i) KeReleaseSpinLock(l, i);
  69 #endif
  70
  71 #define ExAcquireRundownProtection                      _ExAcquireRundownProtection
  72 #define ExReleaseRundownProtection                      _ExReleaseRundownProtection
  73 #define ExInitializeRundownProtection                   _ExInitializeRundownProtection
  74 #define ExWaitForRundownProtectionRelease               _ExWaitForRundownProtectionRelease
  75 #define ExRundownCompleted                              _ExRundownCompleted
  76 #define ExGetPreviousMode                               KeGetPreviousMode
  77
  78
  79 //
  80 // Various bits tagged on the handle or handle table
  81 //
  82 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT    1
  83 #define FREE_HANDLE_MASK                -1
  84
  85 //
  86 // Number of entries in each table level
  87 //
  88 #define LOW_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
  89 #define MID_LEVEL_ENTRIES   (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
  90 #define HIGH_LEVEL_ENTRIES  (65535 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
  91
  92 //
  93 // Maximum index in each table level before we need another table
  94 //
  95 #define MAX_LOW_INDEX       LOW_LEVEL_ENTRIES
  96 #define MAX_MID_INDEX       (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
  97 #define MAX_HIGH_INDEX      (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
  98
  99 //
 100 // Detect GCC 4.1.2+
 101 //
 102 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40102
 103
 104 //
 105 // Broken GCC with Alignment Bug. We'll do alignment ourselves at higher cost.
 106 //
 107 #define DEFINE_WAIT_BLOCK(x)                                \
 108     struct _AlignHack                                       \
 109     {                                                       \
 110         UCHAR Hack[15];                                     \
 111         EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock;             \
 112     } WaitBlockBuffer;                                      \
 113     PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
 114         ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
 115
 116 #else
 117
 118 //
 119 // This is only for compatibility; the compiler will optimize the extra
 120 // local variable (the actual pointer) away, so we don't take any perf hit
 121 // by doing this.
 122 //
 123 #define DEFINE_WAIT_BLOCK(x)                                \
 124     EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer;                \
 125     PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
 126
 127 #endif
 128
 129 /* INITIALIZATION FUNCTIONS *************************************************/
 130
 131 VOID
 132 NTAPI
 133 ExpWin32kInit(VOID);
 134
 135 VOID
 136 NTAPI
 137 ExInit2(VOID);
 138
 139 VOID
 140 NTAPI
 141 ExPhase2Init(
 142     IN PVOID Context
 143 );
 144
 145 VOID
 146 NTAPI
 147 ExpInitializePushLocks(VOID);
 148
 149 BOOLEAN
 150 NTAPI
 151 ExRefreshTimeZoneInformation(
 152     IN PLARGE_INTEGER SystemBootTime
 153 );
 154
 155 VOID
 156 NTAPI
 157 ExpInitializeWorkerThreads(VOID);
 158
 159 VOID
 160 NTAPI
 161 ExpInitLookasideLists(VOID);
 162
 163 VOID
 164 NTAPI
 165 ExInitializeSystemLookasideList(
 166     IN PGENERAL_LOOKASIDE List,
 167     IN POOL_TYPE Type,
 168     IN ULONG Size,
 169     IN ULONG Tag,
 170     IN USHORT MaximumDepth,
 171     IN PLIST_ENTRY ListHead
 172 );
 173
 174 BOOLEAN
 175 NTAPI
 176 ExpInitializeCallbacks(VOID);
 177
 178 VOID
 179 NTAPI
 180 ExpInitUuids(VOID);
 181
 182 VOID
 183 NTAPI
 184 ExpInitializeExecutive(
 185     IN ULONG Cpu,
 186     IN PLOADER_PARAMETER_BLOCK LoaderBlock
 187 );
 188
 189 VOID
 190 NTAPI
 191 ExpInitializeEventImplementation(VOID);
 192
 193 VOID
 194 NTAPI
 195 ExpInitializeEventImplementation(VOID);
 196
 197 VOID
 198 NTAPI
 199 ExpInitializeEventPairImplementation(VOID);
 200
 201 VOID
 202 NTAPI
 203 ExpInitializeSemaphoreImplementation(VOID);
 204
 205 VOID
 206 NTAPI
 207 ExpInitializeMutantImplementation(VOID);
 208
 209 VOID
 210 NTAPI
 211 ExpInitializeTimerImplementation(VOID);
 212
 213 VOID
 214 NTAPI
 215 ExpInitializeProfileImplementation(VOID);
 216
 217 VOID
 218 NTAPI
 219 ExpResourceInitialization(VOID);
 220
 221 VOID
 222 NTAPI
 223 ExInitPoolLookasidePointers(VOID);
 224
 225 /* Callback Functions ********************************************************/
 226
 227 VOID
 228 NTAPI
 229 ExInitializeCallBack(
 230     IN OUT PEX_CALLBACK Callback
 231 );
 232
 233 PEX_CALLBACK_ROUTINE_BLOCK
 234 NTAPI
 235 ExAllocateCallBack(
 236     IN PEX_CALLBACK_FUNCTION Function,
 237     IN PVOID Context
 238 );
 239
 240 VOID
 241 NTAPI
 242 ExFreeCallBack(
 243     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 244 );
 245
 246 BOOLEAN
 247 NTAPI
 248 ExCompareExchangeCallBack (
 249     IN OUT PEX_CALLBACK CallBack,
 250     IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
 251     IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
 252 );
 253
 254 PEX_CALLBACK_ROUTINE_BLOCK
 255 NTAPI
 256 ExReferenceCallBackBlock(
 257     IN OUT PEX_CALLBACK CallBack
 258 );
 259
 260 VOID
 261 NTAPI
 262 ExDereferenceCallBackBlock(
 263     IN OUT PEX_CALLBACK CallBack,
 264     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 265 );
 266
 267 PEX_CALLBACK_FUNCTION
 268 NTAPI
 269 ExGetCallBackBlockRoutine(
 270     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 271 );
 272
 273 PVOID
 274 NTAPI
 275 ExGetCallBackBlockContext(
 276     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 277 );
 278
 279 VOID
 280 NTAPI
 281 ExWaitForCallBacks(
 282     IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
 283 );
 284
 285 /* Rundown Functions ********************************************************/
 286
 287 VOID
 288 FASTCALL
 289 ExfInitializeRundownProtection(
 290      OUT PEX_RUNDOWN_REF RunRef
 291 );
 292
 293 VOID
 294 FASTCALL
 295 ExfReInitializeRundownProtection(
 296      OUT PEX_RUNDOWN_REF RunRef
 297 );
 298
 299 BOOLEAN
 300 FASTCALL
 301 ExfAcquireRundownProtection(
 302      IN OUT PEX_RUNDOWN_REF RunRef
 303 );
 304
 305 BOOLEAN
 306 FASTCALL
 307 ExfAcquireRundownProtectionEx(
 308      IN OUT PEX_RUNDOWN_REF RunRef,
 309      IN ULONG Count
 310 );
 311
 312 VOID
 313 FASTCALL
 314 ExfReleaseRundownProtection(
 315      IN OUT PEX_RUNDOWN_REF RunRef
 316 );
 317
 318 VOID
 319 FASTCALL
 320 ExfReleaseRundownProtectionEx(
 321      IN OUT PEX_RUNDOWN_REF RunRef,
 322      IN ULONG Count
 323 );
 324
 325 VOID
 326 FASTCALL
 327 ExfRundownCompleted(
 328      OUT PEX_RUNDOWN_REF RunRef
 329 );
 330
 331 VOID
 332 FASTCALL
 333 ExfWaitForRundownProtectionRelease(
 334      IN OUT PEX_RUNDOWN_REF RunRef
 335 );
 336
 337 /* HANDLE TABLE FUNCTIONS ***************************************************/
 338
 339 typedef VOID
 340 (NTAPI *PEX_SWEEP_HANDLE_CALLBACK)(
 341     PHANDLE_TABLE_ENTRY HandleTableEntry,
 342     HANDLE Handle,
 343     PVOID Context
 344 );
 345
 346 typedef BOOLEAN
 347 (NTAPI *PEX_DUPLICATE_HANDLE_CALLBACK)(
 348     IN PEPROCESS Process,
 349     IN PHANDLE_TABLE HandleTable,
 350     IN PHANDLE_TABLE_ENTRY HandleTableEntry,
 351     IN PHANDLE_TABLE_ENTRY NewEntry
 352 );
 353
 354 typedef BOOLEAN
 355 (NTAPI *PEX_CHANGE_HANDLE_CALLBACK)(
 356     PHANDLE_TABLE_ENTRY HandleTableEntry,
 357     ULONG_PTR Context
 358 );
 359
 360 VOID
 361 NTAPI
 362 ExpInitializeHandleTables(
 363     VOID
 364 );
 365
 366 PHANDLE_TABLE
 367 NTAPI
 368 ExCreateHandleTable(
 369     IN PEPROCESS Process OPTIONAL
 370 );
 371
 372 VOID
 373 NTAPI
 374 ExUnlockHandleTableEntry(
 375     IN PHANDLE_TABLE HandleTable,
 376     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 377 );
 378
 379 HANDLE
 380 NTAPI
 381 ExCreateHandle(
 382     IN PHANDLE_TABLE HandleTable,
 383     IN PHANDLE_TABLE_ENTRY HandleTableEntry
 384 );
 385
 386 VOID
 387 NTAPI
 388 ExDestroyHandleTable(
 389     IN PHANDLE_TABLE HandleTable,
 390     IN PVOID DestroyHandleProcedure OPTIONAL
 391 );
 392
 393 BOOLEAN
 394 NTAPI
 395 ExDestroyHandle(
 396     IN PHANDLE_TABLE HandleTable,
 397     IN HANDLE Handle,
 398     IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
 399 );
 400
 401 PHANDLE_TABLE_ENTRY
 402 NTAPI
 403 ExMapHandleToPointer(
 404     IN PHANDLE_TABLE HandleTable,
 405     IN HANDLE Handle
 406 );
 407
 408 PHANDLE_TABLE
 409 NTAPI
 410 ExDupHandleTable(
 411     IN PEPROCESS Process,
 412     IN PHANDLE_TABLE HandleTable,
 413     IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure,
 414     IN ULONG_PTR Mask
 415 );
 416
 417 BOOLEAN
 418 NTAPI
 419 ExChangeHandle(
 420     IN PHANDLE_TABLE HandleTable,
 421     IN HANDLE Handle,
 422     IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine,
 423     IN ULONG_PTR Context
 424 );
 425
 426 VOID
 427 NTAPI
 428 ExSweepHandleTable(
 429     IN PHANDLE_TABLE HandleTable,
 430     IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure,
 431     IN PVOID Context
 432 );
 433
 434 /* PSEH EXCEPTION HANDLING **************************************************/
 435
 436 LONG
 437 NTAPI
 438 ExSystemExceptionFilter(VOID);
 439
 440 static __inline _SEH_FILTER(_SEH_ExSystemExceptionFilter)
 441 {
 442     return ExSystemExceptionFilter();
 443 }
 444
 445 /* CALLBACKS *****************************************************************/
 446
 447 VOID
 448 FORCEINLINE
 449 ExDoCallBack(IN OUT PEX_CALLBACK Callback,
 450              IN PVOID Context,
 451              IN PVOID Argument1,
 452              IN PVOID Argument2)
 453 {
 454     PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock;
 455     PEX_CALLBACK_FUNCTION Function;
 456
 457     /* Reference the block */
 458     CallbackRoutineBlock = ExReferenceCallBackBlock(Callback);
 459     if (CallbackRoutineBlock)
 460     {
 461         /* Get the function */
 462         Function = ExGetCallBackBlockRoutine(CallbackRoutineBlock);
 463
 464         /* Do the callback */
 465         Function(Context, Argument1, Argument2);
 466
 467         /* Now dereference it */
 468         ExDereferenceCallBackBlock(Callback, CallbackRoutineBlock);
 469     }
 470 }
 471
 472 /* RUNDOWN *******************************************************************/
 473
 474 #ifdef _WIN64
 475 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
 476 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
 477 #else
 478 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z))
 479 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
 480 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
 481 #endif
 482
 483 /*++
 484  * @name ExfAcquireRundownProtection
 485  * INTERNAL MACRO
 486  *
 487  *     The ExfAcquireRundownProtection routine acquires rundown protection for
 488  *     the specified descriptor.
 489  *
 490  * @param RunRef
 491  *        Pointer to a rundown reference descriptor.
 492  *
 493  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 494  *
 495  * @remarks This is the internal macro for system use only.In case the rundown
 496  *          was active, then the slow-path will be called through the exported
 497  *          function.
 498  *
 499  *--*/
 500 BOOLEAN
 501 FORCEINLINE
 502 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 503 {
 504     ULONG_PTR Value, NewValue;
 505
 506     /* Get the current value and mask the active bit */
 507     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 508
 509     /* Add a reference */
 510     NewValue = Value + EX_RUNDOWN_COUNT_INC;
 511
 512     /* Change the value */
 513     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 514     if (NewValue != Value)
 515     {
 516         /* Rundown was active, use long path */
 517         return ExfAcquireRundownProtection(RunRef);
 518     }
 519
 520     /* Success */
 521     return TRUE;
 522 }
 523
 524 /*++
 525  * @name ExReleaseRundownProtection
 526  * INTERNAL MACRO
 527  *
 528  *     The ExReleaseRundownProtection routine releases rundown protection for
 529  *     the specified descriptor.
 530  *
 531  * @param RunRef
 532  *        Pointer to a rundown reference descriptor.
 533  *
 534  * @return TRUE if access to the protected structure was granted, FALSE otherwise.
 535  *
 536  * @remarks This is the internal macro for system use only.In case the rundown
 537  *          was active, then the slow-path will be called through the exported
 538  *          function.
 539  *
 540  *--*/
 541 VOID
 542 FORCEINLINE
 543 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 544 {
 545     ULONG_PTR Value, NewValue;
 546
 547     /* Get the current value and mask the active bit */
 548     Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
 549
 550     /* Remove a reference */
 551     NewValue = Value - EX_RUNDOWN_COUNT_INC;
 552
 553     /* Change the value */
 554     NewValue = ExpChangeRundown(RunRef, NewValue, Value);
 555
 556     /* Check if the rundown was active */
 557     if (NewValue != Value)
 558     {
 559         /* Rundown was active, use long path */
 560         ExfReleaseRundownProtection(RunRef);
 561     }
 562     else
 563     {
 564         /* Sanity check */
 565         ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
 566     }
 567 }
 568
 569 /*++
 570  * @name ExInitializeRundownProtection
 571  * INTERNAL MACRO
 572  *
 573  *     The ExInitializeRundownProtection routine initializes a rundown
 574  *     protection descriptor.
 575  *
 576  * @param RunRef
 577  *        Pointer to a rundown reference descriptor.
 578  *
 579  * @return None.
 580  *
 581  * @remarks This is the internal macro for system use only.
 582  *
 583  *--*/
 584 VOID
 585 FORCEINLINE
 586 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
 587 {
 588     /* Set the count to zero */
 589     RunRef->Count = 0;
 590 }
 591
 592 /*++
 593  * @name ExWaitForRundownProtectionRelease
 594  * INTERNAL MACRO
 595  *
 596  *     The ExWaitForRundownProtectionRelease routine waits until the specified
 597  *     rundown descriptor has been released.
 598  *
 599  * @param RunRef
 600  *        Pointer to a rundown reference descriptor.
 601  *
 602  * @return None.
 603  *
 604  * @remarks This is the internal macro for system use only. If a wait is actually
 605  *          necessary, then the slow path is taken through the exported function.
 606  *
 607  *--*/
 608 VOID
 609 FORCEINLINE
 610 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
 611 {
 612     ULONG_PTR Value;
 613
 614     /* Set the active bit */
 615     Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
 616     if ((Value) && (Value != EX_RUNDOWN_ACTIVE))
 617     {
 618         /* If the the rundown wasn't already active, then take the long path */
 619         ExfWaitForRundownProtectionRelease(RunRef);
 620     }
 621 }
 622
 623 /*++
 624  * @name ExRundownCompleted
 625  * INTERNAL MACRO
 626  *
 627  *     The ExRundownCompleted routine completes the rundown of the specified
 628  *     descriptor by setting the active bit.
 629  *
 630  * @param RunRef
 631  *        Pointer to a rundown reference descriptor.
 632  *
 633  * @return None.
 634  *
 635  * @remarks This is the internal macro for system use only.
 636  *
 637  *--*/
 638 VOID
 639 FORCEINLINE
 640 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
 641 {
 642     /* Sanity check */
 643     ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 644
 645     /* Mark the counter as active */
 646     ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
 647 }
 648
 649 /* PUSHLOCKS *****************************************************************/
 650
 651 /* FIXME: VERIFY THESE! */
 652
 653 VOID
 654 FASTCALL
 655 ExBlockPushLock(PEX_PUSH_LOCK PushLock,
 656                 PVOID WaitBlock);
 657
 658 VOID
 659 FASTCALL
 660 ExfUnblockPushLock(PEX_PUSH_LOCK PushLock,
 661                    PVOID CurrentWaitBlock);
 662
 663 VOID
 664 FASTCALL
 665 ExWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
 666                          IN PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock);
 667
 668 /*++
 669  * @name ExInitializePushLock
 670  * INTERNAL MACRO
 671  *
 672  *     The ExInitializePushLock macro initializes a PushLock.
 673  *
 674  * @params PushLock
 675  *         Pointer to the pushlock which is to be initialized.
 676  *
 677  * @return None.
 678  *
 679  * @remarks None.
 680  *
 681  *--*/
 682 VOID
 683 FORCEINLINE
 684 ExInitializePushLock(IN PULONG_PTR PushLock)
 685 {
 686     /* Set the value to 0 */
 687     *PushLock = 0;
 688 }
 689
 690 /*++
 691  * @name ExAcquirePushLockExclusive
 692  * INTERNAL MACRO
 693  *
 694  *     The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
 695  *
 696  * @params PushLock
 697  *         Pointer to the pushlock which is to be acquired.
 698  *
 699  * @return None.
 700  *
 701  * @remarks The function attempts the quickest route to acquire the lock, which is
 702  *          to simply set the lock bit.
 703  *          However, if the pushlock is already shared, the slower path is taken.
 704  *
 705  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 706  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 707  *
 708  *--*/
 709 VOID
 710 FORCEINLINE
 711 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
 712 {
 713     /* Try acquiring the lock */
 714     if (InterlockedBitTestAndSet((PLONG)PushLock, EX_PUSH_LOCK_LOCK_V))
 715     {
 716         /* Someone changed it, use the slow path */
 717         DbgPrint("%s - Contention!\n", __FUNCTION__);
 718         ExfAcquirePushLockExclusive(PushLock);
 719     }
 720
 721     /* Sanity check */
 722     ASSERT(PushLock->Locked);
 723 }
 724
 725 /*++
 726  * @name ExAcquirePushLockShared
 727  * INTERNAL MACRO
 728  *
 729  *     The ExAcquirePushLockShared macro acquires a shared PushLock.
 730  *
 731  * @params PushLock
 732  *         Pointer to the pushlock which is to be acquired.
 733  *
 734  * @return None.
 735  *
 736  * @remarks The function attempts the quickest route to acquire the lock, which is
 737  *          to simply set the lock bit and set the share count to one.
 738  *          However, if the pushlock is already shared, the slower path is taken.
 739  *
 740  *          Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
 741  *          This macro should usually be paired up with KeAcquireCriticalRegion.
 742  *
 743  *--*/
 744 VOID
 745 FORCEINLINE
 746 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
 747 {
 748     EX_PUSH_LOCK NewValue;
 749
 750     /* Try acquiring the lock */
 751     NewValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
 752     if (ExpChangePushlock(PushLock, NewValue.Ptr, 0))
 753     {
 754         /* Someone changed it, use the slow path */
 755         DbgPrint("%s - Contention!\n", __FUNCTION__);
 756         ExfAcquirePushLockShared(PushLock);
 757     }
 758
 759     /* Sanity checks */
 760     ASSERT(PushLock->Locked);
 761     ASSERT(PushLock->Waiting || PushLock->Shared > 0);
 762 }
 763
 764 /*++
 765  * @name ExConvertPushLockSharedToExclusive
 766  * INTERNAL MACRO
 767  *
 768  *     The ExConvertPushLockSharedToExclusive macro converts an exclusive
 769  *     pushlock to a shared pushlock.
 770  *
 771  * @params PushLock
 772  *         Pointer to the pushlock which is to be converted.
 773  *
 774  * @return FALSE if conversion failed, TRUE otherwise.
 775  *
 776  * @remarks The function attempts the quickest route to convert the lock, which is
 777  *          to simply set the lock bit and remove any other bits.
 778  *
 779  *--*/
 780 BOOLEAN
 781 FORCEINLINE
 782 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock)
 783 {
 784     EX_PUSH_LOCK OldValue;
 785
 786     /* Set the expected old value */
 787     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
 788
 789     /* Try converting the lock */
 790     if (ExpChangePushlock(PushLock, EX_PUSH_LOCK_LOCK, OldValue.Value) !=
 791         OldValue.Ptr)
 792     {
 793         /* Conversion failed */
 794         return FALSE;
 795     }
 796
 797     /* Sanity check */
 798     ASSERT(PushLock->Locked);
 799     return TRUE;
 800 }
 801
 802 /*++
 803  * @name ExWaitOnPushLock
 804  * INTERNAL MACRO
 805  *
 806  *     The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
 807  *
 808  * @params PushLock
 809  *         Pointer to a pushlock.
 810  *
 811  * @return None.
 812  *
 813  * @remarks The function attempts to get any exclusive waiters out of their slow
 814  *          path by forcing an instant acquire/release operation.
 815  *
 816  *          Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
 817  *
 818  *--*/
 819 VOID
 820 FORCEINLINE
 821 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock)
 822 {
 823     /* Check if we're locked */
 824     if (PushLock->Locked)
 825     {
 826         /* Acquire the lock */
 827         ExfAcquirePushLockExclusive(PushLock);
 828         ASSERT(PushLock->Locked);
 829
 830         /* Release it */
 831         ExfReleasePushLockExclusive(PushLock);
 832     }
 833 }
 834
 835 /*++
 836  * @name ExReleasePushLockShared
 837  * INTERNAL MACRO
 838  *
 839  *     The ExReleasePushLockShared macro releases a previously acquired PushLock.
 840  *
 841  * @params PushLock
 842  *         Pointer to a previously acquired pushlock.
 843  *
 844  * @return None.
 845  *
 846  * @remarks The function attempts the quickest route to release the lock, which is
 847  *          to simply decrease the share count and remove the lock bit.
 848  *          However, if the pushlock is being waited on then the long path is taken.
 849  *
 850  *          Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
 851  *          This macro should usually be paired up with KeLeaveCriticalRegion.
 852  *
 853  *--*/
 854 VOID
 855 FORCEINLINE
 856 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock)
 857 {
 858     EX_PUSH_LOCK OldValue;
 859
 860     /* Sanity checks */
 861     ASSERT(PushLock->Locked);
 862     ASSERT(PushLock->Waiting || PushLock->Shared > 0);
 863
 864     /* Try to clear the pushlock */
 865     OldValue.Value = EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_SHARE_INC;
 866     if (ExpChangePushlock(PushLock, 0, OldValue.Ptr) != OldValue.Ptr)
 867     {
 868         /* There are still other people waiting on it */
 869         DbgPrint("%s - Contention!\n", __FUNCTION__);
 870         ExfReleasePushLockShared(PushLock);
 871     }
 872 }
 873
 874 /*++
 875  * @name ExReleasePushLockExclusive
 876  * INTERNAL MACRO
 877  *
 878  *     The ExReleasePushLockExclusive macro releases a previously
 879  *     exclusively acquired PushLock.
 880  *
 881  * @params PushLock
 882  *         Pointer to a previously acquired pushlock.
 883  *
 884  * @return None.
 885  *
 886  * @remarks The function attempts the quickest route to release the lock, which is
 887  *          to simply clear the locked bit.
 888  *          However, if the pushlock is being waited on, the slow path is taken
 889  *          in an attempt to wake up the lock.
 890  *
 891  *          Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
 892  *          This macro should usually be paired up with KeLeaveCriticalRegion.
 893  *
 894  *--*/
 895 VOID
 896 FORCEINLINE
 897 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
 898 {
 899     EX_PUSH_LOCK OldValue;
 900
 901     /* Sanity checks */
 902     ASSERT(PushLock->Locked);
 903     ASSERT(PushLock->Waiting || PushLock->Shared == 0);
 904
 905     /* Unlock the pushlock */
 906     OldValue.Value = InterlockedExchangeAddSizeT((PLONG)PushLock,
 907                                                  -EX_PUSH_LOCK_LOCK);
 908
 909     /* Sanity checks */
 910     ASSERT(OldValue.Locked);
 911     ASSERT(OldValue.Waiting || OldValue.Shared == 0);
 912
 913     /* Check if anyone is waiting on it and it's not already waking*/
 914     if ((OldValue.Waiting) && !(OldValue.Waking))
 915     {
 916         /* Wake it up */
 917         DbgPrint("%s - Contention!\n", __FUNCTION__);
 918         ExfTryToWakePushLock(PushLock);
 919     }
 920 }
 921
 922 /*++
 923  * @name ExReleasePushLock
 924  * INTERNAL MACRO
 925  *
 926  *     The ExReleasePushLock macro releases a previously acquired PushLock.
 927  *
 928  * @params PushLock
 929  *         Pointer to a previously acquired pushlock.
 930  *
 931  * @return None.
 932  *
 933  * @remarks The function attempts the quickest route to release the lock, which is
 934  *          to simply clear all the fields and decrease the share count if required.
 935  *          However, if the pushlock is being waited on then the long path is taken.
 936  *
 937  *          Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
 938  *          This macro should usually be paired up with KeLeaveCriticalRegion.
 939  *
 940  *--*/
 941 VOID
 942 FORCEINLINE
 943 ExReleasePushLock(PEX_PUSH_LOCK PushLock)
 944 {
 945     EX_PUSH_LOCK OldValue = *PushLock;
 946     EX_PUSH_LOCK NewValue;
 947
 948     /* Sanity checks */
 949     ASSERT(OldValue.Locked);
 950
 951     /* Check if the pushlock is shared */
 952     if (OldValue.Shared > 1)
 953     {
 954         /* Decrease the share count */
 955         NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_SHARE_INC;
 956     }
 957     else
 958     {
 959         /* Clear the pushlock entirely */
 960         NewValue.Value = 0;
 961     }
 962
 963     /* Check if nobody is waiting on us and try clearing the lock here */
 964     if ((OldValue.Waiting) ||
 965         (ExpChangePushlock(PushLock, NewValue.Ptr, OldValue.Ptr) !=
 966          OldValue.Ptr))
 967     {
 968         /* We have waiters, use the long path */
 969         DbgPrint("%s - Contention!\n", __FUNCTION__);
 970         ExfReleasePushLock(PushLock);
 971     }
 972 }
 973
 974 /* OTHER FUNCTIONS **********************************************************/
 975
 976 LONGLONG
 977 FASTCALL
 978 ExfpInterlockedExchange64(
 979     LONGLONG volatile * Destination,
 980     PLONGLONG Exchange
 981 );
 982
 983 NTSTATUS
 984 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation);
 985
 986 NTSTATUS
 987 NTAPI
 988 ExpAllocateLocallyUniqueId(OUT LUID *LocallyUniqueId);
 989
 990 VOID
 991 NTAPI
 992 ExTimerRundown(VOID);
 993
 994 #define InterlockedDecrementUL(Addend) \
 995    (ULONG)InterlockedDecrement((PLONG)(Addend))
 996
 997 #define InterlockedIncrementUL(Addend) \
 998    (ULONG)InterlockedIncrement((PLONG)(Addend))
 999
1000 #define InterlockedExchangeUL(Target, Value) \
1001    (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1002
1003 #define InterlockedExchangeAddUL(Addend, Value) \
1004    (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1005
1006 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1007    (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1008
1009 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1010    (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))
1011
1012 #define ExfpInterlockedExchange64UL(Target, Value) \
1013    (ULONGLONG)ExfpInterlockedExchange64((PLONGLONG)(Target), (PLONGLONG)(Value))
1014
1015 #endif /* __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H */