8 /* GLOBAL VARIABLES *********************************************************/
10 extern RTL_TIME_ZONE_INFORMATION ExpTimeZoneInfo
;
11 extern LARGE_INTEGER ExpTimeZoneBias
;
12 extern ULONG ExpTimeZoneId
;
13 extern ULONG ExpTickCountMultiplier
;
14 extern ULONG ExpLastTimeZoneBias
;
15 extern POBJECT_TYPE ExEventPairObjectType
;
16 extern POBJECT_TYPE _ExEventObjectType
, _ExSemaphoreObjectType
;
17 extern FAST_MUTEX ExpEnvironmentLock
;
18 extern ERESOURCE ExpFirmwareTableResource
;
19 extern ERESOURCE ExpTimeRefreshLock
;
20 extern LIST_ENTRY ExpFirmwareTableProviderListHead
;
21 extern BOOLEAN ExpIsWinPEMode
;
22 extern LIST_ENTRY ExpSystemResourcesList
;
23 extern ULONG ExpAnsiCodePageDataOffset
, ExpOemCodePageDataOffset
;
24 extern ULONG ExpUnicodeCaseTableDataOffset
;
25 extern PVOID ExpNlsSectionPointer
;
26 extern ULONG NtGlobalFlag
;
27 extern UNICODE_STRING NtSystemRoot
;
28 extern ULONG ExpInitializationPhase
;
29 extern ULONG ExpAltTimeZoneBias
;
30 extern LIST_ENTRY ExSystemLookasideListHead
;
31 extern PCALLBACK_OBJECT PowerStateCallback
;
32 extern LIST_ENTRY ExPoolLookasideListHead
;
33 extern LIST_ENTRY ExpNonPagedLookasideListHead
;
34 extern LIST_ENTRY ExpPagedLookasideListHead
;
35 extern KSPIN_LOCK ExpNonPagedLookasideListLock
;
36 extern KSPIN_LOCK ExpPagedLookasideListLock
;
37 extern ULONG ExCriticalWorkerThreads
;
38 extern ULONG ExDelayedWorkerThreads
;
40 extern PVOID ExpDefaultErrorPort
;
41 extern PEPROCESS ExpDefaultErrorPortProcess
;
44 * NT/Cm Version Info variables
46 extern ULONG NtMajorVersion
;
47 extern ULONG NtMinorVersion
;
48 extern ULONG NtBuildNumber
;
49 extern ULONG CmNtSpBuildNumber
;
50 extern ULONG CmNtCSDVersion
;
51 extern ULONG CmNtCSDReleaseType
;
52 extern UNICODE_STRING CmVersionString
;
53 extern UNICODE_STRING CmCSDVersionString
;
54 extern CHAR NtBuildLab
[];
57 * WinDBG Debugger Worker State Machine data (see dbgctrl.c)
59 typedef enum _WINKD_WORKER_STATE
63 WinKdWorkerInitialized
66 extern WORK_QUEUE_ITEM ExpDebuggerWorkItem
;
67 extern WINKD_WORKER_STATE ExpDebuggerWork
;
68 extern PEPROCESS ExpDebuggerProcessAttach
;
69 extern PEPROCESS ExpDebuggerProcessKill
;
70 extern ULONG_PTR ExpDebuggerPageIn
;
72 VOID NTAPI
ExpDebuggerWorker(IN PVOID Context
);
75 #define HANDLE_LOW_BITS (PAGE_SHIFT - 4)
76 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 3)
78 #define HANDLE_LOW_BITS (PAGE_SHIFT - 3)
79 #define HANDLE_HIGH_BITS (PAGE_SHIFT - 2)
81 #define HANDLE_TAG_BITS (2)
82 #define HANDLE_INDEX_BITS (HANDLE_LOW_BITS + 2*HANDLE_HIGH_BITS)
83 #define KERNEL_FLAG_BITS (sizeof(PVOID)*8 - HANDLE_INDEX_BITS - HANDLE_TAG_BITS)
85 typedef union _EXHANDLE
89 ULONG_PTR TagBits
: HANDLE_TAG_BITS
;
90 ULONG_PTR Index
: HANDLE_INDEX_BITS
;
91 ULONG_PTR KernelFlag
: KERNEL_FLAG_BITS
;
95 ULONG_PTR TagBits2
: HANDLE_TAG_BITS
;
96 ULONG_PTR LowIndex
: HANDLE_LOW_BITS
;
97 ULONG_PTR MidIndex
: HANDLE_HIGH_BITS
;
98 ULONG_PTR HighIndex
: HANDLE_HIGH_BITS
;
99 ULONG_PTR KernelFlag2
: KERNEL_FLAG_BITS
;
101 HANDLE GenericHandleOverlay
;
104 } EXHANDLE
, *PEXHANDLE
;
106 typedef struct _ETIMER
111 LIST_ENTRY ActiveTimerListEntry
;
114 BOOLEAN ApcAssociated
;
116 LIST_ENTRY WakeTimerListEntry
;
121 PCALLBACK_OBJECT
*CallbackObject
;
125 typedef struct _HARDERROR_USER_PARAMETERS
127 ULONG_PTR Parameters
[MAXIMUM_HARDERROR_PARAMETERS
];
128 UNICODE_STRING Strings
[MAXIMUM_HARDERROR_PARAMETERS
];
129 WCHAR Buffer
[ANYSIZE_ARRAY
];
130 } HARDERROR_USER_PARAMETERS
, *PHARDERROR_USER_PARAMETERS
;
132 #define MAX_FAST_REFS 7
134 #define ExAcquireRundownProtection _ExAcquireRundownProtection
135 #define ExReleaseRundownProtection _ExReleaseRundownProtection
136 #define ExInitializeRundownProtection _ExInitializeRundownProtection
137 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
138 #define ExRundownCompleted _ExRundownCompleted
139 #define ExGetPreviousMode KeGetPreviousMode
143 // Various bits tagged on the handle or handle table
145 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
146 #define FREE_HANDLE_MASK -1
149 // Number of entries in each table level
151 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
152 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
153 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
156 // Maximum index in each table level before we need another table
158 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
159 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
160 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
162 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
163 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
164 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
169 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation
,
170 IN ULONG SystemInformationLength
,
171 IN OUT PULONG ReturnLength OPTIONAL
174 typedef struct _UUID_CACHED_VALUES_STRUCT
182 UCHAR ClockSeqHiAndReserved
;
184 UCHAR NodeId
[6 /*SEED_BUFFER_SIZE*/];
186 UCHAR GuidInit
[8]; /* Match GUID.Data4 */
188 } UUID_CACHED_VALUES_STRUCT
, *PUUID_CACHED_VALUES_STRUCT
;
190 C_ASSERT(RTL_FIELD_SIZE(UUID_CACHED_VALUES_STRUCT
, GuidInit
) == RTL_FIELD_SIZE(UUID
, Data4
));
192 /* INITIALIZATION FUNCTIONS *************************************************/
205 Phase1Initialization(
212 ExpInitializePushLocks(VOID
);
216 ExRefreshTimeZoneInformation(
217 IN PLARGE_INTEGER SystemBootTime
223 ExpInitializeWorkerThreads(VOID
);
227 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap
);
232 ExpInitLookasideLists(VOID
);
237 ExInitializeSystemLookasideList(
238 IN PGENERAL_LOOKASIDE List
,
242 IN USHORT MaximumDepth
,
243 IN PLIST_ENTRY ListHead
249 ExpInitializeCallbacks(VOID
);
254 ExpUuidInitialization(VOID
);
259 ExLuidInitialization(VOID
);
264 ExpInitializeExecutive(
266 IN PLOADER_PARAMETER_BLOCK LoaderBlock
271 ExShutdownSystem(VOID
);
276 ExpInitializeEventImplementation(VOID
);
281 ExpInitializeKeyedEventImplementation(VOID
);
286 ExpInitializeEventPairImplementation(VOID
);
291 ExpInitializeSemaphoreImplementation(VOID
);
296 ExpInitializeMutantImplementation(VOID
);
301 ExpInitializeTimerImplementation(VOID
);
306 ExpInitializeProfileImplementation(VOID
);
311 ExpResourceInitialization(VOID
);
316 ExInitPoolLookasidePointers(VOID
);
318 /* Callback Functions ********************************************************/
322 ExInitializeCallBack(
323 IN OUT PEX_CALLBACK Callback
326 PEX_CALLBACK_ROUTINE_BLOCK
329 IN PEX_CALLBACK_FUNCTION Function
,
336 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
341 ExCompareExchangeCallBack (
342 IN OUT PEX_CALLBACK CallBack
,
343 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock
,
344 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
347 PEX_CALLBACK_ROUTINE_BLOCK
349 ExReferenceCallBackBlock(
350 IN OUT PEX_CALLBACK CallBack
355 ExDereferenceCallBackBlock(
356 IN OUT PEX_CALLBACK CallBack
,
357 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
360 PEX_CALLBACK_FUNCTION
362 ExGetCallBackBlockRoutine(
363 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
368 ExGetCallBackBlockContext(
369 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
375 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
378 /* Rundown Functions ********************************************************/
382 ExfInitializeRundownProtection(
383 OUT PEX_RUNDOWN_REF RunRef
388 ExfReInitializeRundownProtection(
389 OUT PEX_RUNDOWN_REF RunRef
394 ExfAcquireRundownProtection(
395 IN OUT PEX_RUNDOWN_REF RunRef
400 ExfAcquireRundownProtectionEx(
401 IN OUT PEX_RUNDOWN_REF RunRef
,
407 ExfReleaseRundownProtection(
408 IN OUT PEX_RUNDOWN_REF RunRef
413 ExfReleaseRundownProtectionEx(
414 IN OUT PEX_RUNDOWN_REF RunRef
,
421 OUT PEX_RUNDOWN_REF RunRef
426 ExfWaitForRundownProtectionRelease(
427 IN OUT PEX_RUNDOWN_REF RunRef
430 /* HANDLE TABLE FUNCTIONS ***************************************************/
433 (NTAPI
*PEX_SWEEP_HANDLE_CALLBACK
)(
434 PHANDLE_TABLE_ENTRY HandleTableEntry
,
440 (NTAPI
*PEX_DUPLICATE_HANDLE_CALLBACK
)(
441 IN PEPROCESS Process
,
442 IN PHANDLE_TABLE HandleTable
,
443 IN PHANDLE_TABLE_ENTRY HandleTableEntry
,
444 IN PHANDLE_TABLE_ENTRY NewEntry
448 (NTAPI
*PEX_CHANGE_HANDLE_CALLBACK
)(
449 PHANDLE_TABLE_ENTRY HandleTableEntry
,
456 ExpInitializeHandleTables(
463 IN PEPROCESS Process OPTIONAL
468 ExUnlockHandleTableEntry(
469 IN PHANDLE_TABLE HandleTable
,
470 IN PHANDLE_TABLE_ENTRY HandleTableEntry
476 IN PHANDLE_TABLE HandleTable
,
477 IN PHANDLE_TABLE_ENTRY HandleTableEntry
482 ExDestroyHandleTable(
483 IN PHANDLE_TABLE HandleTable
,
484 IN PVOID DestroyHandleProcedure OPTIONAL
490 IN PHANDLE_TABLE HandleTable
,
492 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
497 ExMapHandleToPointer(
498 IN PHANDLE_TABLE HandleTable
,
505 IN PEPROCESS Process
,
506 IN PHANDLE_TABLE HandleTable
,
507 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure
,
514 IN PHANDLE_TABLE HandleTable
,
516 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine
,
523 IN PHANDLE_TABLE HandleTable
,
524 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure
,
530 ExpLookupHandleTableEntry(
531 IN PHANDLE_TABLE HandleTable
,
537 ExpLockHandleTableEntry(
538 IN PHANDLE_TABLE HandleTable
,
539 IN PHANDLE_TABLE_ENTRY HandleTableEntry
542 /* PSEH EXCEPTION HANDLING **************************************************/
546 ExSystemExceptionFilter(VOID
);
548 /* CALLBACKS *****************************************************************/
552 ExDoCallBack(IN OUT PEX_CALLBACK Callback
,
557 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock
;
558 PEX_CALLBACK_FUNCTION Function
;
560 /* Reference the block */
561 CallbackBlock
= ExReferenceCallBackBlock(Callback
);
564 /* Get the function */
565 Function
= ExGetCallBackBlockRoutine(CallbackBlock
);
567 /* Do the callback */
568 Function(Context
, Argument1
, Argument2
);
570 /* Now dereference it */
571 ExDereferenceCallBackBlock(Callback
, CallbackBlock
);
575 /* FAST REFS ******************************************************************/
579 ExGetObjectFastReference(IN EX_FAST_REF FastRef
)
581 /* Return the unbiased pointer */
582 return (PVOID
)(FastRef
.Value
& ~MAX_FAST_REFS
);
587 ExGetCountFastReference(IN EX_FAST_REF FastRef
)
589 /* Return the reference count */
590 return (ULONG
)FastRef
.RefCnt
;
595 ExInitializeFastReference(OUT PEX_FAST_REF FastRef
,
596 IN OPTIONAL PVOID Object
)
599 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
601 /* Check if an object is being set */
604 /* Clear the field */
605 FastRef
->Object
= NULL
;
609 /* Otherwise, we assume the object was referenced and is ready */
610 FastRef
->Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
616 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef
)
618 EX_FAST_REF OldValue
, NewValue
;
620 /* Start reference loop */
623 /* Get the current reference count */
627 /* Increase the reference count */
628 NewValue
.Value
= OldValue
.Value
- 1;
629 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
632 if (NewValue
.Object
!= OldValue
.Object
) continue;
639 /* Return the old value */
645 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef
,
648 EX_FAST_REF OldValue
, NewValue
;
651 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
653 /* Start update loop */
656 /* Get the current reference count */
659 /* Check if the current count is too high or if the pointer changed */
660 if (((OldValue
.RefCnt
+ MAX_FAST_REFS
) > MAX_FAST_REFS
) ||
661 ((OldValue
.Value
&~ MAX_FAST_REFS
) != (ULONG_PTR
)Object
))
667 /* Update the reference count */
668 NewValue
.Value
= OldValue
.Value
+ MAX_FAST_REFS
;
669 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
672 if (NewValue
.Object
!= OldValue
.Object
) continue;
684 ExReleaseFastReference(IN PEX_FAST_REF FastRef
,
687 EX_FAST_REF OldValue
, NewValue
;
690 ASSERT(Object
!= NULL
);
691 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
693 /* Start reference loop */
696 /* Get the current reference count */
699 /* Check if we're full if if the pointer changed */
700 if ((OldValue
.Value
^ (ULONG_PTR
)Object
) >= MAX_FAST_REFS
) return FALSE
;
702 /* Decrease the reference count */
703 NewValue
.Value
= OldValue
.Value
+ 1;
704 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
707 if (NewValue
.Object
!= OldValue
.Object
) continue;
719 ExSwapFastReference(IN PEX_FAST_REF FastRef
,
722 EX_FAST_REF NewValue
, OldValue
;
725 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
727 /* Check if an object is being set */
730 /* Clear the field */
731 NewValue
.Object
= NULL
;
735 /* Otherwise, we assume the object was referenced and is ready */
736 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
739 /* Update the object */
740 OldValue
.Object
= InterlockedExchangePointer(&FastRef
->Object
, NewValue
.Object
);
746 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef
,
750 EX_FAST_REF OldValue
, NewValue
;
752 /* Sanity check and start swap loop */
753 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
756 /* Get the current value */
759 /* Make sure there's enough references to swap */
760 if (!((OldValue
.Value
^ (ULONG_PTR
)OldObject
) <= MAX_FAST_REFS
)) break;
762 /* Check if we have an object to swap */
765 /* Set up the value with maximum fast references */
766 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
770 /* Write the object address itself (which is empty) */
771 NewValue
.Value
= (ULONG_PTR
)Object
;
774 /* Do the actual compare exchange */
775 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
778 if (NewValue
.Object
!= OldValue
.Object
) continue;
784 /* Return the old value */
788 /* RUNDOWN *******************************************************************/
792 ExGetRunRefForGivenProcessor(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware
,
795 return (PEX_RUNDOWN_REF
)((ULONG_PTR
)RunRefCacheAware
->RunRefs
+
796 RunRefCacheAware
->RunRefSize
*
797 (ProcNumber
% RunRefCacheAware
->Number
));
801 * @name ExfAcquireRundownProtection
804 * The ExfAcquireRundownProtection routine acquires rundown protection for
805 * the specified descriptor.
808 * Pointer to a rundown reference descriptor.
810 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
812 * @remarks This is the internal macro for system use only.In case the rundown
813 * was active, then the slow-path will be called through the exported
819 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
821 ULONG_PTR Value
, NewValue
;
823 /* Get the current value and mask the active bit */
824 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
826 /* Add a reference */
827 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
829 /* Change the value */
830 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
831 if (NewValue
!= Value
)
833 /* Rundown was active, use long path */
834 return ExfAcquireRundownProtection(RunRef
);
842 * @name ExReleaseRundownProtection
845 * The ExReleaseRundownProtection routine releases rundown protection for
846 * the specified descriptor.
849 * Pointer to a rundown reference descriptor.
851 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
853 * @remarks This is the internal macro for system use only.In case the rundown
854 * was active, then the slow-path will be called through the exported
860 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
862 ULONG_PTR Value
, NewValue
;
864 /* Get the current value and mask the active bit */
865 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
867 /* Remove a reference */
868 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
870 /* Change the value */
871 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
873 /* Check if the rundown was active */
874 if (NewValue
!= Value
)
876 /* Rundown was active, use long path */
877 ExfReleaseRundownProtection(RunRef
);
882 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
887 * @name ExInitializeRundownProtection
890 * The ExInitializeRundownProtection routine initializes a rundown
891 * protection descriptor.
894 * Pointer to a rundown reference descriptor.
898 * @remarks This is the internal macro for system use only.
903 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
905 /* Set the count to zero */
910 * @name ExWaitForRundownProtectionRelease
913 * The ExWaitForRundownProtectionRelease routine waits until the specified
914 * rundown descriptor has been released.
917 * Pointer to a rundown reference descriptor.
921 * @remarks This is the internal macro for system use only. If a wait is actually
922 * necessary, then the slow path is taken through the exported function.
927 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
931 /* Set the active bit */
932 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
933 if ((Value
) && (Value
!= EX_RUNDOWN_ACTIVE
))
935 /* If the the rundown wasn't already active, then take the long path */
936 ExfWaitForRundownProtectionRelease(RunRef
);
941 * @name ExRundownCompleted
944 * The ExRundownCompleted routine completes the rundown of the specified
945 * descriptor by setting the active bit.
948 * Pointer to a rundown reference descriptor.
952 * @remarks This is the internal macro for system use only.
957 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
960 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
962 /* Mark the counter as active */
963 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
966 /* PUSHLOCKS *****************************************************************/
968 /* FIXME: VERIFY THESE! */
973 IN PEX_PUSH_LOCK PushLock
,
980 IN PEX_PUSH_LOCK PushLock
,
981 IN PVOID CurrentWaitBlock
986 ExWaitForUnblockPushLock(
987 IN PEX_PUSH_LOCK PushLock
,
992 * @name _ExInitializePushLock
995 * The _ExInitializePushLock macro initializes a PushLock.
998 * Pointer to the pushlock which is to be initialized.
1007 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock
)
1009 /* Set the value to 0 */
1012 #define ExInitializePushLock _ExInitializePushLock
1015 * @name ExAcquirePushLockExclusive
1018 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1021 * Pointer to the pushlock which is to be acquired.
1025 * @remarks The function attempts the quickest route to acquire the lock, which is
1026 * to simply set the lock bit.
1027 * However, if the pushlock is already shared, the slower path is taken.
1029 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1030 * This macro should usually be paired up with KeAcquireCriticalRegion.
1035 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1037 /* Try acquiring the lock */
1038 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
1040 /* Someone changed it, use the slow path */
1041 ExfAcquirePushLockExclusive(PushLock
);
1045 ASSERT(PushLock
->Locked
);
1049 * @name ExTryToAcquirePushLockExclusive
1052 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
1055 * Pointer to the pushlock which is to be acquired.
1059 * @remarks The function attempts the quickest route to acquire the lock, which is
1060 * to simply set the lock bit.
1061 * However, if the pushlock is already shared, the slower path is taken.
1063 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1064 * This macro should usually be paired up with KeAcquireCriticalRegion.
1069 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1071 /* Try acquiring the lock */
1072 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
1079 ASSERT (PushLock
->Locked
);
1084 * @name ExAcquirePushLockShared
1087 * The ExAcquirePushLockShared macro acquires a shared PushLock.
1090 * Pointer to the pushlock which is to be acquired.
1094 * @remarks The function attempts the quickest route to acquire the lock, which is
1095 * to simply set the lock bit and set the share count to one.
1096 * However, if the pushlock is already shared, the slower path is taken.
1098 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
1099 * This macro should usually be paired up with KeAcquireCriticalRegion.
1104 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
1106 EX_PUSH_LOCK NewValue
;
1108 /* Try acquiring the lock */
1109 NewValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1110 if (ExpChangePushlock(PushLock
, NewValue
.Ptr
, 0))
1112 /* Someone changed it, use the slow path */
1113 ExfAcquirePushLockShared(PushLock
);
1117 ASSERT(PushLock
->Locked
);
1121 * @name ExConvertPushLockSharedToExclusive
1124 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1125 * pushlock to a shared pushlock.
1128 * Pointer to the pushlock which is to be converted.
1130 * @return FALSE if conversion failed, TRUE otherwise.
1132 * @remarks The function attempts the quickest route to convert the lock, which is
1133 * to simply set the lock bit and remove any other bits.
1138 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock
)
1140 EX_PUSH_LOCK OldValue
;
1142 /* Set the expected old value */
1143 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1145 /* Try converting the lock */
1146 if (ExpChangePushlock(PushLock
, EX_PUSH_LOCK_LOCK
, OldValue
.Value
) !=
1149 /* Conversion failed */
1154 ASSERT(PushLock
->Locked
);
1159 * @name ExWaitOnPushLock
1162 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1165 * Pointer to a pushlock.
1169 * @remarks The function attempts to get any exclusive waiters out of their slow
1170 * path by forcing an instant acquire/release operation.
1172 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1177 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock
)
1179 /* Check if we're locked */
1180 if (PushLock
->Locked
)
1182 /* Acquire the lock */
1183 ExfAcquirePushLockExclusive(PushLock
);
1184 ASSERT(PushLock
->Locked
);
1187 ExfReleasePushLockExclusive(PushLock
);
1192 * @name ExReleasePushLockShared
1195 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1198 * Pointer to a previously acquired pushlock.
1202 * @remarks The function attempts the quickest route to release the lock, which is
1203 * to simply decrease the share count and remove the lock bit.
1204 * However, if the pushlock is being waited on then the long path is taken.
1206 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1207 * This macro should usually be paired up with KeLeaveCriticalRegion.
1212 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
1214 EX_PUSH_LOCK OldValue
;
1217 ASSERT(PushLock
->Locked
);
1219 /* Try to clear the pushlock */
1220 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1221 if (ExpChangePushlock(PushLock
, 0, OldValue
.Ptr
) != OldValue
.Ptr
)
1223 /* There are still other people waiting on it */
1224 ExfReleasePushLockShared(PushLock
);
1229 * @name ExReleasePushLockExclusive
1232 * The ExReleasePushLockExclusive macro releases a previously
1233 * exclusively acquired PushLock.
1236 * Pointer to a previously acquired pushlock.
1240 * @remarks The function attempts the quickest route to release the lock, which is
1241 * to simply clear the locked bit.
1242 * However, if the pushlock is being waited on, the slow path is taken
1243 * in an attempt to wake up the lock.
1245 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1246 * This macro should usually be paired up with KeLeaveCriticalRegion.
1251 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1253 EX_PUSH_LOCK OldValue
;
1256 ASSERT(PushLock
->Locked
);
1258 /* Unlock the pushlock */
1259 OldValue
.Value
= InterlockedExchangeAddSizeT((PSIZE_T
)PushLock
,
1260 -(SSIZE_T
)EX_PUSH_LOCK_LOCK
);
1263 ASSERT(OldValue
.Locked
);
1264 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1266 /* Check if anyone is waiting on it and it's not already waking*/
1267 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1270 ExfTryToWakePushLock(PushLock
);
1275 * @name ExReleasePushLock
1278 * The ExReleasePushLock macro releases a previously acquired PushLock.
1281 * Pointer to a previously acquired pushlock.
1285 * @remarks The function attempts the quickest route to release the lock, which is
1286 * to simply clear all the fields and decrease the share count if required.
1287 * However, if the pushlock is being waited on then the long path is taken.
1289 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1290 * This macro should usually be paired up with KeLeaveCriticalRegion.
1295 ExReleasePushLock(PEX_PUSH_LOCK PushLock
)
1297 EX_PUSH_LOCK OldValue
= *PushLock
;
1298 EX_PUSH_LOCK NewValue
;
1301 ASSERT(OldValue
.Locked
);
1303 /* Check if the pushlock is shared */
1304 if (OldValue
.Shared
> 1)
1306 /* Decrease the share count */
1307 NewValue
.Value
= OldValue
.Value
- EX_PUSH_LOCK_SHARE_INC
;
1311 /* Clear the pushlock entirely */
1315 /* Check if nobody is waiting on us and try clearing the lock here */
1316 if ((OldValue
.Waiting
) ||
1317 (ExpChangePushlock(PushLock
, NewValue
.Ptr
, OldValue
.Ptr
) !=
1320 /* We have waiters, use the long path */
1321 ExfReleasePushLock(PushLock
);
1325 /* FAST MUTEX INLINES *********************************************************/
1329 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex
)
1331 PKTHREAD Thread
= KeGetCurrentThread();
1334 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1335 (Thread
->CombinedApcDisable
!= 0) ||
1336 (Thread
->Teb
== NULL
) ||
1337 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1338 ASSERT(FastMutex
->Owner
!= Thread
);
1340 /* Decrease the count */
1341 if (InterlockedDecrement(&FastMutex
->Count
))
1343 /* Someone is still holding it, use slow path */
1344 KiAcquireFastMutex(FastMutex
);
1348 FastMutex
->Owner
= Thread
;
1353 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1355 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1356 (KeGetCurrentThread()->CombinedApcDisable
!= 0) ||
1357 (KeGetCurrentThread()->Teb
== NULL
) ||
1358 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1359 ASSERT(FastMutex
->Owner
== KeGetCurrentThread());
1361 /* Erase the owner */
1362 FastMutex
->Owner
= NULL
;
1364 /* Increase the count */
1365 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1367 /* Someone was waiting for it, signal the waiter */
1368 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1374 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex
)
1377 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1379 /* Raise IRQL to APC */
1380 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1382 /* Decrease the count */
1383 if (InterlockedDecrement(&FastMutex
->Count
))
1385 /* Someone is still holding it, use slow path */
1386 KiAcquireFastMutex(FastMutex
);
1389 /* Set the owner and IRQL */
1390 FastMutex
->Owner
= KeGetCurrentThread();
1391 FastMutex
->OldIrql
= OldIrql
;
1396 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1399 ASSERT(KeGetCurrentIrql() == APC_LEVEL
);
1401 /* Erase the owner */
1402 FastMutex
->Owner
= NULL
;
1403 OldIrql
= (KIRQL
)FastMutex
->OldIrql
;
1405 /* Increase the count */
1406 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1408 /* Someone was waiting for it, signal the waiter */
1409 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1412 /* Lower IRQL back */
1413 KeLowerIrql(OldIrql
);
1418 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1421 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1423 /* Raise to APC_LEVEL */
1424 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1426 /* Check if we can quickly acquire it */
1427 if (InterlockedCompareExchange(&FastMutex
->Count
, 0, 1) == 1)
1429 /* We have, set us as owners */
1430 FastMutex
->Owner
= KeGetCurrentThread();
1431 FastMutex
->OldIrql
= OldIrql
;
1436 /* Acquire attempt failed */
1437 KeLowerIrql(OldIrql
);
1445 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1447 /* Enter the Critical Region */
1448 KeEnterCriticalRegion();
1450 /* Acquire the mutex unsafely */
1451 _ExAcquireFastMutexUnsafe(FastMutex
);
1456 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex
)
1458 /* Release the mutex unsafely */
1459 _ExReleaseFastMutexUnsafe(FastMutex
);
1461 /* Leave the critical region */
1462 KeLeaveCriticalRegion();
1465 /* OTHER FUNCTIONS **********************************************************/
1469 ExTryToAcquireResourceExclusiveLite(
1470 IN PERESOURCE Resource
1474 ExpSetTimeZoneInformation(
1475 IN PRTL_TIME_ZONE_INFORMATION TimeZoneInformation
1480 ExAcquireTimeRefreshLock(
1486 ExReleaseTimeRefreshLock(
1492 ExUpdateSystemTimeFromCmos(
1493 IN BOOLEAN UpdateInterruptTime
,
1494 IN ULONG MaxSepInSeconds
1499 ExAllocateLocallyUniqueId(
1500 OUT LUID
*LocallyUniqueId
1513 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1520 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1523 #define InterlockedDecrementUL(Addend) \
1524 (ULONG)InterlockedDecrement((PLONG)(Addend))
1526 #define InterlockedIncrementUL(Addend) \
1527 (ULONG)InterlockedIncrement((PLONG)(Addend))
1529 #define InterlockedExchangeUL(Target, Value) \
1530 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1532 #define InterlockedExchangeAddUL(Addend, Value) \
1533 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1535 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1536 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1538 #define InterlockedCompareExchangeSizeT(Destination, Exchange, Comperand) \
1539 (SIZE_T)InterlockedCompareExchangePointer((PVOID*)(Destination), (PVOID)(SIZE_T)(Exchange), (PVOID)(SIZE_T)(Comperand))
1542 #define InterlockedExchangeSizeT(Target, Value) \
1543 (SIZE_T)InterlockedExchange64((PLONG64)Target, (LONG64)Value)
1545 #define InterlockedExchangeSizeT(Target, Value) \
1546 (SIZE_T)InterlockedExchange((PLONG)Target, (LONG)Value)
1549 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1550 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))