3 /* GLOBAL VARIABLES *********************************************************/
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo
;
6 extern LARGE_INTEGER ExpTimeZoneBias
;
7 extern ULONG ExpTimeZoneId
;
8 extern ULONG ExpTickCountMultiplier
;
9 extern ULONG ExpLastTimeZoneBias
;
10 extern POBJECT_TYPE ExEventPairObjectType
;
11 extern POBJECT_TYPE _ExEventObjectType
, _ExSemaphoreObjectType
;
12 extern ULONG NtBuildNumber
;
13 extern ULONG NtMajorVersion
;
14 extern ULONG NtMinorVersion
;
15 extern FAST_MUTEX ExpEnvironmentLock
;
16 extern ERESOURCE ExpFirmwareTableResource
;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead
;
18 extern BOOLEAN ExpIsWinPEMode
;
19 extern LIST_ENTRY ExpSystemResourcesList
;
20 extern ULONG ExpAnsiCodePageDataOffset
, ExpOemCodePageDataOffset
;
21 extern ULONG ExpUnicodeCaseTableDataOffset
;
22 extern PVOID ExpNlsSectionPointer
;
23 extern CHAR NtBuildLab
[];
24 extern ULONG CmNtCSDVersion
;
25 extern ULONG NtGlobalFlag
;
26 extern ULONG ExpInitializationPhase
;
27 extern ULONG ExpAltTimeZoneBias
;
28 extern LIST_ENTRY ExSystemLookasideListHead
;
29 extern PCALLBACK_OBJECT PowerStateCallback
;
31 typedef struct _EXHANDLE
40 HANDLE GenericHandleOverlay
;
43 } EXHANDLE
, *PEXHANDLE
;
45 typedef struct _ETIMER
50 LIST_ENTRY ActiveTimerListEntry
;
53 BOOLEAN ApcAssociated
;
55 LIST_ENTRY WakeTimerListEntry
;
60 PCALLBACK_OBJECT
*CallbackObject
;
64 typedef struct _HARDERROR_USER_PARAMETERS
66 ULONG_PTR Parameters
[MAXIMUM_HARDERROR_PARAMETERS
];
67 UNICODE_STRING Strings
[MAXIMUM_HARDERROR_PARAMETERS
];
68 WCHAR Buffer
[ANYSIZE_ARRAY
];
69 } HARDERROR_USER_PARAMETERS
, *PHARDERROR_USER_PARAMETERS
;
71 #define MAX_FAST_REFS 7
73 #define ExAcquireRundownProtection _ExAcquireRundownProtection
74 #define ExReleaseRundownProtection _ExReleaseRundownProtection
75 #define ExInitializeRundownProtection _ExInitializeRundownProtection
76 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
77 #define ExRundownCompleted _ExRundownCompleted
78 #define ExGetPreviousMode KeGetPreviousMode
82 // Various bits tagged on the handle or handle table
84 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
85 #define FREE_HANDLE_MASK -1
88 // Number of entries in each table level
90 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
91 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
92 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
95 // Maximum index in each table level before we need another table
97 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
98 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
99 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
104 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
105 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
107 #define DEFINE_WAIT_BLOCK(x) \
111 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
113 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
114 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
119 // This is only for compatibility; the compiler will optimize the extra
120 // local variable (the actual pointer) away, so we don't take any perf hit
123 #define DEFINE_WAIT_BLOCK(x) \
124 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
125 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
129 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
130 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
131 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
133 /* INITIALIZATION FUNCTIONS *************************************************/
145 Phase1Initialization(
151 ExpInitializePushLocks(VOID
);
155 ExRefreshTimeZoneInformation(
156 IN PLARGE_INTEGER SystemBootTime
161 ExpInitializeWorkerThreads(VOID
);
165 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap
);
169 ExpInitLookasideLists(VOID
);
173 ExInitializeSystemLookasideList(
174 IN PGENERAL_LOOKASIDE List
,
178 IN USHORT MaximumDepth
,
179 IN PLIST_ENTRY ListHead
184 ExpInitializeCallbacks(VOID
);
192 ExpInitializeExecutive(
194 IN PLOADER_PARAMETER_BLOCK LoaderBlock
199 ExpInitializeEventImplementation(VOID
);
203 ExpInitializeEventImplementation(VOID
);
207 ExpInitializeEventPairImplementation(VOID
);
211 ExpInitializeSemaphoreImplementation(VOID
);
215 ExpInitializeMutantImplementation(VOID
);
219 ExpInitializeTimerImplementation(VOID
);
223 ExpInitializeProfileImplementation(VOID
);
227 ExpResourceInitialization(VOID
);
231 ExInitPoolLookasidePointers(VOID
);
233 /* Callback Functions ********************************************************/
237 ExInitializeCallBack(
238 IN OUT PEX_CALLBACK Callback
241 PEX_CALLBACK_ROUTINE_BLOCK
244 IN PEX_CALLBACK_FUNCTION Function
,
251 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
256 ExCompareExchangeCallBack (
257 IN OUT PEX_CALLBACK CallBack
,
258 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock
,
259 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
262 PEX_CALLBACK_ROUTINE_BLOCK
264 ExReferenceCallBackBlock(
265 IN OUT PEX_CALLBACK CallBack
270 ExDereferenceCallBackBlock(
271 IN OUT PEX_CALLBACK CallBack
,
272 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
275 PEX_CALLBACK_FUNCTION
277 ExGetCallBackBlockRoutine(
278 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
283 ExGetCallBackBlockContext(
284 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
290 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
293 /* Rundown Functions ********************************************************/
297 ExfInitializeRundownProtection(
298 OUT PEX_RUNDOWN_REF RunRef
303 ExfReInitializeRundownProtection(
304 OUT PEX_RUNDOWN_REF RunRef
309 ExfAcquireRundownProtection(
310 IN OUT PEX_RUNDOWN_REF RunRef
315 ExfAcquireRundownProtectionEx(
316 IN OUT PEX_RUNDOWN_REF RunRef
,
322 ExfReleaseRundownProtection(
323 IN OUT PEX_RUNDOWN_REF RunRef
328 ExfReleaseRundownProtectionEx(
329 IN OUT PEX_RUNDOWN_REF RunRef
,
336 OUT PEX_RUNDOWN_REF RunRef
341 ExfWaitForRundownProtectionRelease(
342 IN OUT PEX_RUNDOWN_REF RunRef
345 /* HANDLE TABLE FUNCTIONS ***************************************************/
348 (NTAPI
*PEX_SWEEP_HANDLE_CALLBACK
)(
349 PHANDLE_TABLE_ENTRY HandleTableEntry
,
355 (NTAPI
*PEX_DUPLICATE_HANDLE_CALLBACK
)(
356 IN PEPROCESS Process
,
357 IN PHANDLE_TABLE HandleTable
,
358 IN PHANDLE_TABLE_ENTRY HandleTableEntry
,
359 IN PHANDLE_TABLE_ENTRY NewEntry
363 (NTAPI
*PEX_CHANGE_HANDLE_CALLBACK
)(
364 PHANDLE_TABLE_ENTRY HandleTableEntry
,
370 ExpInitializeHandleTables(
377 IN PEPROCESS Process OPTIONAL
382 ExUnlockHandleTableEntry(
383 IN PHANDLE_TABLE HandleTable
,
384 IN PHANDLE_TABLE_ENTRY HandleTableEntry
390 IN PHANDLE_TABLE HandleTable
,
391 IN PHANDLE_TABLE_ENTRY HandleTableEntry
396 ExDestroyHandleTable(
397 IN PHANDLE_TABLE HandleTable
,
398 IN PVOID DestroyHandleProcedure OPTIONAL
404 IN PHANDLE_TABLE HandleTable
,
406 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
411 ExMapHandleToPointer(
412 IN PHANDLE_TABLE HandleTable
,
419 IN PEPROCESS Process
,
420 IN PHANDLE_TABLE HandleTable
,
421 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure
,
428 IN PHANDLE_TABLE HandleTable
,
430 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine
,
437 IN PHANDLE_TABLE HandleTable
,
438 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure
,
442 /* PSEH EXCEPTION HANDLING **************************************************/
446 ExSystemExceptionFilter(VOID
);
448 /* CALLBACKS *****************************************************************/
452 ExDoCallBack(IN OUT PEX_CALLBACK Callback
,
457 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock
;
458 PEX_CALLBACK_FUNCTION Function
;
460 /* Reference the block */
461 CallbackBlock
= ExReferenceCallBackBlock(Callback
);
464 /* Get the function */
465 Function
= ExGetCallBackBlockRoutine(CallbackBlock
);
467 /* Do the callback */
468 Function(Context
, Argument1
, Argument2
);
470 /* Now dereference it */
471 ExDereferenceCallBackBlock(Callback
, CallbackBlock
);
475 /* FAST REFS ******************************************************************/
479 ExGetObjectFastReference(IN EX_FAST_REF FastRef
)
481 /* Return the unbiased pointer */
482 return (PVOID
)(FastRef
.Value
& ~MAX_FAST_REFS
);
487 ExGetCountFastReference(IN EX_FAST_REF FastRef
)
489 /* Return the reference count */
490 return (ULONG
)FastRef
.RefCnt
;
495 ExInitializeFastReference(OUT PEX_FAST_REF FastRef
,
496 IN OPTIONAL PVOID Object
)
499 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
501 /* Check if an object is being set */
504 /* Clear the field */
505 FastRef
->Object
= NULL
;
509 /* Otherwise, we assume the object was referenced and is ready */
510 FastRef
->Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
516 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef
)
518 EX_FAST_REF OldValue
, NewValue
;
520 /* Start reference loop */
523 /* Get the current reference count */
527 /* Increase the reference count */
528 NewValue
.Value
= OldValue
.Value
- 1;
529 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
532 if (NewValue
.Object
!= OldValue
.Object
) continue;
539 /* Return the old value */
545 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef
,
548 EX_FAST_REF OldValue
, NewValue
;
551 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
553 /* Start update loop */
556 /* Get the current reference count */
559 /* Check if the current count is too high or if the pointer changed */
560 if (((OldValue
.RefCnt
+ MAX_FAST_REFS
) > MAX_FAST_REFS
) ||
561 ((OldValue
.Value
&~ MAX_FAST_REFS
) != (ULONG_PTR
)Object
))
567 /* Update the reference count */
568 NewValue
.Value
= OldValue
.Value
+ MAX_FAST_REFS
;
569 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
572 if (NewValue
.Object
!= OldValue
.Object
) continue;
584 ExReleaseFastReference(IN PEX_FAST_REF FastRef
,
587 EX_FAST_REF OldValue
, NewValue
;
590 ASSERT(Object
!= NULL
);
591 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
593 /* Start reference loop */
596 /* Get the current reference count */
599 /* Check if we're full if if the pointer changed */
600 if ((OldValue
.Value
^ (ULONG_PTR
)Object
) >= MAX_FAST_REFS
) return FALSE
;
602 /* Decrease the reference count */
603 NewValue
.Value
= OldValue
.Value
+ 1;
604 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
607 if (NewValue
.Object
!= OldValue
.Object
) continue;
619 ExSwapFastReference(IN PEX_FAST_REF FastRef
,
622 EX_FAST_REF NewValue
, OldValue
;
625 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
627 /* Check if an object is being set */
630 /* Clear the field */
631 NewValue
.Object
= NULL
;
635 /* Otherwise, we assume the object was referenced and is ready */
636 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
639 /* Update the object */
640 OldValue
.Object
= InterlockedExchangePointer(&FastRef
->Object
, NewValue
.Object
);
646 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef
,
650 EX_FAST_REF OldValue
, NewValue
;
652 /* Sanity check and start swap loop */
653 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
656 /* Get the current value */
659 /* Make sure there's enough references to swap */
660 if (!((OldValue
.Value
^ (ULONG_PTR
)OldObject
) <= MAX_FAST_REFS
)) break;
662 /* Check if we have an object to swap */
665 /* Set up the value with maximum fast references */
666 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
670 /* Write the object address itself (which is empty) */
671 NewValue
.Value
= (ULONG_PTR
)Object
;
674 /* Do the actual compare exchange */
675 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
678 if (NewValue
.Object
!= OldValue
.Object
) continue;
684 /* Return the old value */
688 /* RUNDOWN *******************************************************************/
691 * @name ExfAcquireRundownProtection
694 * The ExfAcquireRundownProtection routine acquires rundown protection for
695 * the specified descriptor.
698 * Pointer to a rundown reference descriptor.
700 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
702 * @remarks This is the internal macro for system use only.In case the rundown
703 * was active, then the slow-path will be called through the exported
709 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
711 ULONG_PTR Value
, NewValue
;
713 /* Get the current value and mask the active bit */
714 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
716 /* Add a reference */
717 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
719 /* Change the value */
720 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
721 if (NewValue
!= Value
)
723 /* Rundown was active, use long path */
724 return ExfAcquireRundownProtection(RunRef
);
732 * @name ExReleaseRundownProtection
735 * The ExReleaseRundownProtection routine releases rundown protection for
736 * the specified descriptor.
739 * Pointer to a rundown reference descriptor.
741 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
743 * @remarks This is the internal macro for system use only.In case the rundown
744 * was active, then the slow-path will be called through the exported
750 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
752 ULONG_PTR Value
, NewValue
;
754 /* Get the current value and mask the active bit */
755 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
757 /* Remove a reference */
758 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
760 /* Change the value */
761 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
763 /* Check if the rundown was active */
764 if (NewValue
!= Value
)
766 /* Rundown was active, use long path */
767 ExfReleaseRundownProtection(RunRef
);
772 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
777 * @name ExInitializeRundownProtection
780 * The ExInitializeRundownProtection routine initializes a rundown
781 * protection descriptor.
784 * Pointer to a rundown reference descriptor.
788 * @remarks This is the internal macro for system use only.
793 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
795 /* Set the count to zero */
800 * @name ExWaitForRundownProtectionRelease
803 * The ExWaitForRundownProtectionRelease routine waits until the specified
804 * rundown descriptor has been released.
807 * Pointer to a rundown reference descriptor.
811 * @remarks This is the internal macro for system use only. If a wait is actually
812 * necessary, then the slow path is taken through the exported function.
817 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
821 /* Set the active bit */
822 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
823 if ((Value
) && (Value
!= EX_RUNDOWN_ACTIVE
))
825 /* If the the rundown wasn't already active, then take the long path */
826 ExfWaitForRundownProtectionRelease(RunRef
);
831 * @name ExRundownCompleted
834 * The ExRundownCompleted routine completes the rundown of the specified
835 * descriptor by setting the active bit.
838 * Pointer to a rundown reference descriptor.
842 * @remarks This is the internal macro for system use only.
847 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
850 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
852 /* Mark the counter as active */
853 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
856 /* PUSHLOCKS *****************************************************************/
858 /* FIXME: VERIFY THESE! */
863 IN PEX_PUSH_LOCK PushLock
,
870 IN PEX_PUSH_LOCK PushLock
,
871 IN PVOID CurrentWaitBlock
876 ExWaitForUnblockPushLock(
877 IN PEX_PUSH_LOCK PushLock
,
882 * @name _ExInitializePushLock
885 * The _ExInitializePushLock macro initializes a PushLock.
888 * Pointer to the pushlock which is to be initialized.
897 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock
)
899 /* Set the value to 0 */
902 #define ExInitializePushLock _ExInitializePushLock
905 * @name ExAcquirePushLockExclusive
908 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
911 * Pointer to the pushlock which is to be acquired.
915 * @remarks The function attempts the quickest route to acquire the lock, which is
916 * to simply set the lock bit.
917 * However, if the pushlock is already shared, the slower path is taken.
919 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
920 * This macro should usually be paired up with KeAcquireCriticalRegion.
925 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
927 /* Try acquiring the lock */
928 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
930 /* Someone changed it, use the slow path */
931 ExfAcquirePushLockExclusive(PushLock
);
935 ASSERT(PushLock
->Locked
);
939 * @name ExTryToAcquirePushLockExclusive
942 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
945 * Pointer to the pushlock which is to be acquired.
949 * @remarks The function attempts the quickest route to acquire the lock, which is
950 * to simply set the lock bit.
951 * However, if the pushlock is already shared, the slower path is taken.
953 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
954 * This macro should usually be paired up with KeAcquireCriticalRegion.
959 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
961 /* Try acquiring the lock */
962 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
969 ASSERT (PushLock
->Locked
);
974 * @name ExAcquirePushLockShared
977 * The ExAcquirePushLockShared macro acquires a shared PushLock.
980 * Pointer to the pushlock which is to be acquired.
984 * @remarks The function attempts the quickest route to acquire the lock, which is
985 * to simply set the lock bit and set the share count to one.
986 * However, if the pushlock is already shared, the slower path is taken.
988 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
989 * This macro should usually be paired up with KeAcquireCriticalRegion.
994 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
996 EX_PUSH_LOCK NewValue
;
998 /* Try acquiring the lock */
999 NewValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1000 if (ExpChangePushlock(PushLock
, NewValue
.Ptr
, 0))
1002 /* Someone changed it, use the slow path */
1003 ExfAcquirePushLockShared(PushLock
);
1007 ASSERT(PushLock
->Locked
);
1008 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1012 * @name ExConvertPushLockSharedToExclusive
1015 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1016 * pushlock to a shared pushlock.
1019 * Pointer to the pushlock which is to be converted.
1021 * @return FALSE if conversion failed, TRUE otherwise.
1023 * @remarks The function attempts the quickest route to convert the lock, which is
1024 * to simply set the lock bit and remove any other bits.
1029 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock
)
1031 EX_PUSH_LOCK OldValue
;
1033 /* Set the expected old value */
1034 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1036 /* Try converting the lock */
1037 if (ExpChangePushlock(PushLock
, EX_PUSH_LOCK_LOCK
, OldValue
.Value
) !=
1040 /* Conversion failed */
1045 ASSERT(PushLock
->Locked
);
1050 * @name ExWaitOnPushLock
1053 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1056 * Pointer to a pushlock.
1060 * @remarks The function attempts to get any exclusive waiters out of their slow
1061 * path by forcing an instant acquire/release operation.
1063 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1068 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock
)
1070 /* Check if we're locked */
1071 if (PushLock
->Locked
)
1073 /* Acquire the lock */
1074 ExfAcquirePushLockExclusive(PushLock
);
1075 ASSERT(PushLock
->Locked
);
1078 ExfReleasePushLockExclusive(PushLock
);
1083 * @name ExReleasePushLockShared
1086 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1089 * Pointer to a previously acquired pushlock.
1093 * @remarks The function attempts the quickest route to release the lock, which is
1094 * to simply decrease the share count and remove the lock bit.
1095 * However, if the pushlock is being waited on then the long path is taken.
1097 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1098 * This macro should usually be paired up with KeLeaveCriticalRegion.
1103 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
1105 EX_PUSH_LOCK OldValue
;
1108 ASSERT(PushLock
->Locked
);
1109 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1111 /* Try to clear the pushlock */
1112 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1113 if (ExpChangePushlock(PushLock
, 0, OldValue
.Ptr
) != OldValue
.Ptr
)
1115 /* There are still other people waiting on it */
1116 ExfReleasePushLockShared(PushLock
);
1121 * @name ExReleasePushLockExclusive
1124 * The ExReleasePushLockExclusive macro releases a previously
1125 * exclusively acquired PushLock.
1128 * Pointer to a previously acquired pushlock.
1132 * @remarks The function attempts the quickest route to release the lock, which is
1133 * to simply clear the locked bit.
1134 * However, if the pushlock is being waited on, the slow path is taken
1135 * in an attempt to wake up the lock.
1137 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1138 * This macro should usually be paired up with KeLeaveCriticalRegion.
1143 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1145 EX_PUSH_LOCK OldValue
;
1148 ASSERT(PushLock
->Locked
);
1149 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
== 0);
1151 /* Unlock the pushlock */
1152 OldValue
.Value
= InterlockedExchangeAddSizeT((PSIZE_T
)PushLock
,
1153 -(SSIZE_T
)EX_PUSH_LOCK_LOCK
);
1156 ASSERT(OldValue
.Locked
);
1157 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1159 /* Check if anyone is waiting on it and it's not already waking*/
1160 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1163 ExfTryToWakePushLock(PushLock
);
1168 * @name ExReleasePushLock
1171 * The ExReleasePushLock macro releases a previously acquired PushLock.
1174 * Pointer to a previously acquired pushlock.
1178 * @remarks The function attempts the quickest route to release the lock, which is
1179 * to simply clear all the fields and decrease the share count if required.
1180 * However, if the pushlock is being waited on then the long path is taken.
1182 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1183 * This macro should usually be paired up with KeLeaveCriticalRegion.
1188 ExReleasePushLock(PEX_PUSH_LOCK PushLock
)
1190 EX_PUSH_LOCK OldValue
= *PushLock
;
1191 EX_PUSH_LOCK NewValue
;
1194 ASSERT(OldValue
.Locked
);
1196 /* Check if the pushlock is shared */
1197 if (OldValue
.Shared
> 1)
1199 /* Decrease the share count */
1200 NewValue
.Value
= OldValue
.Value
- EX_PUSH_LOCK_SHARE_INC
;
1204 /* Clear the pushlock entirely */
1208 /* Check if nobody is waiting on us and try clearing the lock here */
1209 if ((OldValue
.Waiting
) ||
1210 (ExpChangePushlock(PushLock
, NewValue
.Ptr
, OldValue
.Ptr
) !=
1213 /* We have waiters, use the long path */
1214 ExfReleasePushLock(PushLock
);
1218 /* FAST MUTEX INLINES *********************************************************/
1222 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex
)
1224 PKTHREAD Thread
= KeGetCurrentThread();
1227 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1228 (Thread
->CombinedApcDisable
!= 0) ||
1229 (Thread
->Teb
== NULL
) ||
1230 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1231 ASSERT(FastMutex
->Owner
!= Thread
);
1233 /* Decrease the count */
1234 if (InterlockedDecrement(&FastMutex
->Count
))
1236 /* Someone is still holding it, use slow path */
1237 KiAcquireFastMutex(FastMutex
);
1241 FastMutex
->Owner
= Thread
;
1246 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1248 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1249 (KeGetCurrentThread()->CombinedApcDisable
!= 0) ||
1250 (KeGetCurrentThread()->Teb
== NULL
) ||
1251 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1252 ASSERT(FastMutex
->Owner
== KeGetCurrentThread());
1254 /* Erase the owner */
1255 FastMutex
->Owner
= NULL
;
1257 /* Increase the count */
1258 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1260 /* Someone was waiting for it, signal the waiter */
1261 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1267 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex
)
1270 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1272 /* Raise IRQL to APC */
1273 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1275 /* Decrease the count */
1276 if (InterlockedDecrement(&FastMutex
->Count
))
1278 /* Someone is still holding it, use slow path */
1279 KiAcquireFastMutex(FastMutex
);
1282 /* Set the owner and IRQL */
1283 FastMutex
->Owner
= KeGetCurrentThread();
1284 FastMutex
->OldIrql
= OldIrql
;
1289 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1292 ASSERT(KeGetCurrentIrql() == APC_LEVEL
);
1294 /* Erase the owner */
1295 FastMutex
->Owner
= NULL
;
1296 OldIrql
= (KIRQL
)FastMutex
->OldIrql
;
1298 /* Increase the count */
1299 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1301 /* Someone was waiting for it, signal the waiter */
1302 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1305 /* Lower IRQL back */
1306 KeLowerIrql(OldIrql
);
1311 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1314 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1316 /* Raise to APC_LEVEL */
1317 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1319 /* Check if we can quickly acquire it */
1320 if (InterlockedCompareExchange(&FastMutex
->Count
, 0, 1) == 1)
1322 /* We have, set us as owners */
1323 FastMutex
->Owner
= KeGetCurrentThread();
1324 FastMutex
->OldIrql
= OldIrql
;
1329 /* Acquire attempt failed */
1330 KeLowerIrql(OldIrql
);
1338 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1340 /* Enter the Critical Region */
1341 KeEnterCriticalRegion();
1343 /* Acquire the mutex unsafely */
1344 _ExAcquireFastMutexUnsafe(FastMutex
);
1349 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex
)
1351 /* Release the mutex unsafely */
1352 _ExReleaseFastMutexUnsafe(FastMutex
);
1354 /* Leave the critical region */
1355 KeLeaveCriticalRegion();
1358 /* OTHER FUNCTIONS **********************************************************/
1362 ExTryToAcquireResourceExclusiveLite(
1363 IN PERESOURCE Resource
1367 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation
);
1371 ExAcquireTimeRefreshLock(BOOLEAN Wait
);
1375 ExReleaseTimeRefreshLock(VOID
);
1379 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime
,
1380 IN ULONG MaxSepInSeconds
);
1384 ExpAllocateLocallyUniqueId(OUT LUID
*LocallyUniqueId
);
1388 ExTimerRundown(VOID
);
1393 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1399 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1402 #define InterlockedDecrementUL(Addend) \
1403 (ULONG)InterlockedDecrement((PLONG)(Addend))
1405 #define InterlockedIncrementUL(Addend) \
1406 (ULONG)InterlockedIncrement((PLONG)(Addend))
1408 #define InterlockedExchangeUL(Target, Value) \
1409 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1411 #define InterlockedExchangeAddUL(Addend, Value) \
1412 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1414 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1415 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1417 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1418 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))