3 /* GLOBAL VARIABLES *********************************************************/
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo
;
6 extern LARGE_INTEGER ExpTimeZoneBias
;
7 extern ULONG ExpTimeZoneId
;
8 extern ULONG ExpTickCountMultiplier
;
9 extern ULONG ExpLastTimeZoneBias
;
10 extern POBJECT_TYPE ExEventPairObjectType
;
11 extern POBJECT_TYPE _ExEventObjectType
, _ExSemaphoreObjectType
;
12 extern ULONG NtBuildNumber
;
13 extern ULONG NtMajorVersion
;
14 extern ULONG NtMinorVersion
;
15 extern FAST_MUTEX ExpEnvironmentLock
;
16 extern ERESOURCE ExpFirmwareTableResource
;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead
;
18 extern BOOLEAN ExpIsWinPEMode
;
19 extern LIST_ENTRY ExpSystemResourcesList
;
20 extern ULONG ExpAnsiCodePageDataOffset
, ExpOemCodePageDataOffset
;
21 extern ULONG ExpUnicodeCaseTableDataOffset
;
22 extern PVOID ExpNlsSectionPointer
;
23 extern CHAR NtBuildLab
[];
24 extern ULONG CmNtCSDVersion
;
25 extern ULONG NtGlobalFlag
;
26 extern ULONG ExpInitializationPhase
;
27 extern ULONG ExpAltTimeZoneBias
;
28 extern LIST_ENTRY ExSystemLookasideListHead
;
29 extern PCALLBACK_OBJECT PowerStateCallback
;
31 typedef struct _EXHANDLE
40 HANDLE GenericHandleOverlay
;
43 } EXHANDLE
, *PEXHANDLE
;
45 typedef struct _ETIMER
50 LIST_ENTRY ActiveTimerListEntry
;
53 BOOLEAN ApcAssociated
;
55 LIST_ENTRY WakeTimerListEntry
;
60 PCALLBACK_OBJECT
*CallbackObject
;
64 #define MAX_FAST_REFS 7
66 #define ExAcquireRundownProtection _ExAcquireRundownProtection
67 #define ExReleaseRundownProtection _ExReleaseRundownProtection
68 #define ExInitializeRundownProtection _ExInitializeRundownProtection
69 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
70 #define ExRundownCompleted _ExRundownCompleted
71 #define ExGetPreviousMode KeGetPreviousMode
75 // Various bits tagged on the handle or handle table
77 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
78 #define FREE_HANDLE_MASK -1
81 // Number of entries in each table level
83 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
84 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
85 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
88 // Maximum index in each table level before we need another table
90 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
91 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
92 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
97 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
98 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
100 #define DEFINE_WAIT_BLOCK(x) \
104 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
106 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
107 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
112 // This is only for compatibility; the compiler will optimize the extra
113 // local variable (the actual pointer) away, so we don't take any perf hit
116 #define DEFINE_WAIT_BLOCK(x) \
117 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
118 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
123 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
124 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
125 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
127 #define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
128 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
129 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
132 /* INITIALIZATION FUNCTIONS *************************************************/
144 Phase1Initialization(
150 ExpInitializePushLocks(VOID
);
154 ExRefreshTimeZoneInformation(
155 IN PLARGE_INTEGER SystemBootTime
160 ExpInitializeWorkerThreads(VOID
);
164 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap
);
168 ExpInitLookasideLists(VOID
);
172 ExInitializeSystemLookasideList(
173 IN PGENERAL_LOOKASIDE List
,
177 IN USHORT MaximumDepth
,
178 IN PLIST_ENTRY ListHead
183 ExpInitializeCallbacks(VOID
);
191 ExpInitializeExecutive(
193 IN PLOADER_PARAMETER_BLOCK LoaderBlock
198 ExpInitializeEventImplementation(VOID
);
202 ExpInitializeEventImplementation(VOID
);
206 ExpInitializeEventPairImplementation(VOID
);
210 ExpInitializeSemaphoreImplementation(VOID
);
214 ExpInitializeMutantImplementation(VOID
);
218 ExpInitializeTimerImplementation(VOID
);
222 ExpInitializeProfileImplementation(VOID
);
226 ExpResourceInitialization(VOID
);
230 ExInitPoolLookasidePointers(VOID
);
232 /* Callback Functions ********************************************************/
236 ExInitializeCallBack(
237 IN OUT PEX_CALLBACK Callback
240 PEX_CALLBACK_ROUTINE_BLOCK
243 IN PEX_CALLBACK_FUNCTION Function
,
250 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
255 ExCompareExchangeCallBack (
256 IN OUT PEX_CALLBACK CallBack
,
257 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock
,
258 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
261 PEX_CALLBACK_ROUTINE_BLOCK
263 ExReferenceCallBackBlock(
264 IN OUT PEX_CALLBACK CallBack
269 ExDereferenceCallBackBlock(
270 IN OUT PEX_CALLBACK CallBack
,
271 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
274 PEX_CALLBACK_FUNCTION
276 ExGetCallBackBlockRoutine(
277 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
282 ExGetCallBackBlockContext(
283 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
289 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
292 /* Rundown Functions ********************************************************/
296 ExfInitializeRundownProtection(
297 OUT PEX_RUNDOWN_REF RunRef
302 ExfReInitializeRundownProtection(
303 OUT PEX_RUNDOWN_REF RunRef
308 ExfAcquireRundownProtection(
309 IN OUT PEX_RUNDOWN_REF RunRef
314 ExfAcquireRundownProtectionEx(
315 IN OUT PEX_RUNDOWN_REF RunRef
,
321 ExfReleaseRundownProtection(
322 IN OUT PEX_RUNDOWN_REF RunRef
327 ExfReleaseRundownProtectionEx(
328 IN OUT PEX_RUNDOWN_REF RunRef
,
335 OUT PEX_RUNDOWN_REF RunRef
340 ExfWaitForRundownProtectionRelease(
341 IN OUT PEX_RUNDOWN_REF RunRef
344 /* HANDLE TABLE FUNCTIONS ***************************************************/
347 (NTAPI
*PEX_SWEEP_HANDLE_CALLBACK
)(
348 PHANDLE_TABLE_ENTRY HandleTableEntry
,
354 (NTAPI
*PEX_DUPLICATE_HANDLE_CALLBACK
)(
355 IN PEPROCESS Process
,
356 IN PHANDLE_TABLE HandleTable
,
357 IN PHANDLE_TABLE_ENTRY HandleTableEntry
,
358 IN PHANDLE_TABLE_ENTRY NewEntry
362 (NTAPI
*PEX_CHANGE_HANDLE_CALLBACK
)(
363 PHANDLE_TABLE_ENTRY HandleTableEntry
,
369 ExpInitializeHandleTables(
376 IN PEPROCESS Process OPTIONAL
381 ExUnlockHandleTableEntry(
382 IN PHANDLE_TABLE HandleTable
,
383 IN PHANDLE_TABLE_ENTRY HandleTableEntry
389 IN PHANDLE_TABLE HandleTable
,
390 IN PHANDLE_TABLE_ENTRY HandleTableEntry
395 ExDestroyHandleTable(
396 IN PHANDLE_TABLE HandleTable
,
397 IN PVOID DestroyHandleProcedure OPTIONAL
403 IN PHANDLE_TABLE HandleTable
,
405 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
410 ExMapHandleToPointer(
411 IN PHANDLE_TABLE HandleTable
,
418 IN PEPROCESS Process
,
419 IN PHANDLE_TABLE HandleTable
,
420 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure
,
427 IN PHANDLE_TABLE HandleTable
,
429 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine
,
436 IN PHANDLE_TABLE HandleTable
,
437 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure
,
441 /* PSEH EXCEPTION HANDLING **************************************************/
445 ExSystemExceptionFilter(VOID
);
447 /* CALLBACKS *****************************************************************/
451 ExDoCallBack(IN OUT PEX_CALLBACK Callback
,
456 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock
;
457 PEX_CALLBACK_FUNCTION Function
;
459 /* Reference the block */
460 CallbackBlock
= ExReferenceCallBackBlock(Callback
);
463 /* Get the function */
464 Function
= ExGetCallBackBlockRoutine(CallbackBlock
);
466 /* Do the callback */
467 Function(Context
, Argument1
, Argument2
);
469 /* Now dereference it */
470 ExDereferenceCallBackBlock(Callback
, CallbackBlock
);
474 /* FAST REFS ******************************************************************/
478 ExGetObjectFastReference(IN EX_FAST_REF FastRef
)
480 /* Return the unbiased pointer */
481 return (PVOID
)(FastRef
.Value
& ~MAX_FAST_REFS
);
486 ExGetCountFastReference(IN EX_FAST_REF FastRef
)
488 /* Return the reference count */
489 return FastRef
.RefCnt
;
494 ExInitializeFastReference(OUT PEX_FAST_REF FastRef
,
495 IN OPTIONAL PVOID Object
)
498 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
500 /* Check if an object is being set */
503 /* Clear the field */
504 FastRef
->Object
= NULL
;
508 /* Otherwise, we assume the object was referenced and is ready */
509 FastRef
->Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
515 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef
)
517 EX_FAST_REF OldValue
, NewValue
;
519 /* Start reference loop */
522 /* Get the current reference count */
526 /* Increase the reference count */
527 NewValue
.Value
= OldValue
.Value
- 1;
528 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
531 if (NewValue
.Object
!= OldValue
.Object
) continue;
538 /* Return the old value */
544 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef
,
547 EX_FAST_REF OldValue
, NewValue
;
550 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
552 /* Start update loop */
555 /* Get the current reference count */
558 /* Check if the current count is too high or if the pointer changed */
559 if (((OldValue
.RefCnt
+ MAX_FAST_REFS
) > MAX_FAST_REFS
) ||
560 ((OldValue
.Value
&~ MAX_FAST_REFS
) != (ULONG_PTR
)Object
))
566 /* Update the reference count */
567 NewValue
.Value
= OldValue
.Value
+ MAX_FAST_REFS
;
568 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
571 if (NewValue
.Object
!= OldValue
.Object
) continue;
583 ExReleaseFastReference(IN PEX_FAST_REF FastRef
,
586 EX_FAST_REF OldValue
, NewValue
;
589 ASSERT(Object
!= NULL
);
590 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
592 /* Start reference loop */
595 /* Get the current reference count */
598 /* Check if we're full if if the pointer changed */
599 if ((OldValue
.Value
^ (ULONG_PTR
)Object
) >= MAX_FAST_REFS
) return FALSE
;
601 /* Decrease the reference count */
602 NewValue
.Value
= OldValue
.Value
+ 1;
603 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
606 if (NewValue
.Object
!= OldValue
.Object
) continue;
618 ExSwapFastReference(IN PEX_FAST_REF FastRef
,
621 EX_FAST_REF NewValue
, OldValue
;
624 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
626 /* Check if an object is being set */
629 /* Clear the field */
630 NewValue
.Object
= NULL
;
634 /* Otherwise, we assume the object was referenced and is ready */
635 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
638 /* Update the object */
639 OldValue
.Object
= InterlockedExchangePointer(&FastRef
->Object
, NewValue
.Object
);
645 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef
,
649 EX_FAST_REF OldValue
, NewValue
;
651 /* Sanity check and start swap loop */
652 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
655 /* Get the current value */
658 /* Make sure there's enough references to swap */
659 if (!((OldValue
.Value
^ (ULONG_PTR
)OldObject
) <= MAX_FAST_REFS
)) break;
661 /* Check if we have an object to swap */
664 /* Set up the value with maximum fast references */
665 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
669 /* Write the object address itself (which is empty) */
670 NewValue
.Value
= (ULONG_PTR
)Object
;
673 /* Do the actual compare exchange */
674 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
677 if (NewValue
.Object
!= OldValue
.Object
) continue;
683 /* Return the old value */
687 /* RUNDOWN *******************************************************************/
690 * @name ExfAcquireRundownProtection
693 * The ExfAcquireRundownProtection routine acquires rundown protection for
694 * the specified descriptor.
697 * Pointer to a rundown reference descriptor.
699 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
701 * @remarks This is the internal macro for system use only.In case the rundown
702 * was active, then the slow-path will be called through the exported
708 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
710 ULONG_PTR Value
, NewValue
;
712 /* Get the current value and mask the active bit */
713 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
715 /* Add a reference */
716 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
718 /* Change the value */
719 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
720 if (NewValue
!= Value
)
722 /* Rundown was active, use long path */
723 return ExfAcquireRundownProtection(RunRef
);
731 * @name ExReleaseRundownProtection
734 * The ExReleaseRundownProtection routine releases rundown protection for
735 * the specified descriptor.
738 * Pointer to a rundown reference descriptor.
740 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
742 * @remarks This is the internal macro for system use only.In case the rundown
743 * was active, then the slow-path will be called through the exported
749 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
751 ULONG_PTR Value
, NewValue
;
753 /* Get the current value and mask the active bit */
754 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
756 /* Remove a reference */
757 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
759 /* Change the value */
760 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
762 /* Check if the rundown was active */
763 if (NewValue
!= Value
)
765 /* Rundown was active, use long path */
766 ExfReleaseRundownProtection(RunRef
);
771 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
776 * @name ExInitializeRundownProtection
779 * The ExInitializeRundownProtection routine initializes a rundown
780 * protection descriptor.
783 * Pointer to a rundown reference descriptor.
787 * @remarks This is the internal macro for system use only.
792 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
794 /* Set the count to zero */
799 * @name ExWaitForRundownProtectionRelease
802 * The ExWaitForRundownProtectionRelease routine waits until the specified
803 * rundown descriptor has been released.
806 * Pointer to a rundown reference descriptor.
810 * @remarks This is the internal macro for system use only. If a wait is actually
811 * necessary, then the slow path is taken through the exported function.
816 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
820 /* Set the active bit */
821 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
822 if ((Value
) && (Value
!= EX_RUNDOWN_ACTIVE
))
824 /* If the the rundown wasn't already active, then take the long path */
825 ExfWaitForRundownProtectionRelease(RunRef
);
830 * @name ExRundownCompleted
833 * The ExRundownCompleted routine completes the rundown of the specified
834 * descriptor by setting the active bit.
837 * Pointer to a rundown reference descriptor.
841 * @remarks This is the internal macro for system use only.
846 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
849 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
851 /* Mark the counter as active */
852 ExpSetRundown(&RunRef
->Count
, EX_RUNDOWN_ACTIVE
);
855 /* PUSHLOCKS *****************************************************************/
857 /* FIXME: VERIFY THESE! */
862 IN PEX_PUSH_LOCK PushLock
,
869 IN PEX_PUSH_LOCK PushLock
,
870 IN PVOID CurrentWaitBlock
875 ExWaitForUnblockPushLock(
876 IN PEX_PUSH_LOCK PushLock
,
881 * @name _ExInitializePushLock
884 * The _ExInitializePushLock macro initializes a PushLock.
887 * Pointer to the pushlock which is to be initialized.
896 _ExInitializePushLock(IN PULONG_PTR PushLock
)
898 /* Set the value to 0 */
901 #define ExInitializePushLock _ExInitializePushLock
904 * @name ExAcquirePushLockExclusive
907 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
910 * Pointer to the pushlock which is to be acquired.
914 * @remarks The function attempts the quickest route to acquire the lock, which is
915 * to simply set the lock bit.
916 * However, if the pushlock is already shared, the slower path is taken.
918 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
919 * This macro should usually be paired up with KeAcquireCriticalRegion.
924 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
926 /* Try acquiring the lock */
927 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
929 /* Someone changed it, use the slow path */
930 ExfAcquirePushLockExclusive(PushLock
);
934 ASSERT(PushLock
->Locked
);
938 * @name ExTryToAcquirePushLockExclusive
941 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
944 * Pointer to the pushlock which is to be acquired.
948 * @remarks The function attempts the quickest route to acquire the lock, which is
949 * to simply set the lock bit.
950 * However, if the pushlock is already shared, the slower path is taken.
952 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
953 * This macro should usually be paired up with KeAcquireCriticalRegion.
958 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
960 /* Try acquiring the lock */
961 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
968 ASSERT (PushLock
->Locked
);
973 * @name ExAcquirePushLockShared
976 * The ExAcquirePushLockShared macro acquires a shared PushLock.
979 * Pointer to the pushlock which is to be acquired.
983 * @remarks The function attempts the quickest route to acquire the lock, which is
984 * to simply set the lock bit and set the share count to one.
985 * However, if the pushlock is already shared, the slower path is taken.
987 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
988 * This macro should usually be paired up with KeAcquireCriticalRegion.
993 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
995 EX_PUSH_LOCK NewValue
;
997 /* Try acquiring the lock */
998 NewValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
999 if (ExpChangePushlock(PushLock
, NewValue
.Ptr
, 0))
1001 /* Someone changed it, use the slow path */
1002 ExfAcquirePushLockShared(PushLock
);
1006 ASSERT(PushLock
->Locked
);
1007 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1011 * @name ExConvertPushLockSharedToExclusive
1014 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1015 * pushlock to a shared pushlock.
1018 * Pointer to the pushlock which is to be converted.
1020 * @return FALSE if conversion failed, TRUE otherwise.
1022 * @remarks The function attempts the quickest route to convert the lock, which is
1023 * to simply set the lock bit and remove any other bits.
1028 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock
)
1030 EX_PUSH_LOCK OldValue
;
1032 /* Set the expected old value */
1033 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1035 /* Try converting the lock */
1036 if (ExpChangePushlock(PushLock
, EX_PUSH_LOCK_LOCK
, OldValue
.Value
) !=
1039 /* Conversion failed */
1044 ASSERT(PushLock
->Locked
);
1049 * @name ExWaitOnPushLock
1052 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1055 * Pointer to a pushlock.
1059 * @remarks The function attempts to get any exclusive waiters out of their slow
1060 * path by forcing an instant acquire/release operation.
1062 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1067 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock
)
1069 /* Check if we're locked */
1070 if (PushLock
->Locked
)
1072 /* Acquire the lock */
1073 ExfAcquirePushLockExclusive(PushLock
);
1074 ASSERT(PushLock
->Locked
);
1077 ExfReleasePushLockExclusive(PushLock
);
1082 * @name ExReleasePushLockShared
1085 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1088 * Pointer to a previously acquired pushlock.
1092 * @remarks The function attempts the quickest route to release the lock, which is
1093 * to simply decrease the share count and remove the lock bit.
1094 * However, if the pushlock is being waited on then the long path is taken.
1096 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1097 * This macro should usually be paired up with KeLeaveCriticalRegion.
1102 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
1104 EX_PUSH_LOCK OldValue
;
1107 ASSERT(PushLock
->Locked
);
1108 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1110 /* Try to clear the pushlock */
1111 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1112 if (ExpChangePushlock(PushLock
, 0, OldValue
.Ptr
) != OldValue
.Ptr
)
1114 /* There are still other people waiting on it */
1115 ExfReleasePushLockShared(PushLock
);
1120 * @name ExReleasePushLockExclusive
1123 * The ExReleasePushLockExclusive macro releases a previously
1124 * exclusively acquired PushLock.
1127 * Pointer to a previously acquired pushlock.
1131 * @remarks The function attempts the quickest route to release the lock, which is
1132 * to simply clear the locked bit.
1133 * However, if the pushlock is being waited on, the slow path is taken
1134 * in an attempt to wake up the lock.
1136 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1137 * This macro should usually be paired up with KeLeaveCriticalRegion.
1142 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1144 EX_PUSH_LOCK OldValue
;
1147 ASSERT(PushLock
->Locked
);
1148 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
== 0);
1150 /* Unlock the pushlock */
1151 OldValue
.Value
= InterlockedExchangeAddSizeT((PSIZE_T
)PushLock
,
1152 -(SSIZE_T
)EX_PUSH_LOCK_LOCK
);
1155 ASSERT(OldValue
.Locked
);
1156 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1158 /* Check if anyone is waiting on it and it's not already waking*/
1159 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1162 ExfTryToWakePushLock(PushLock
);
1167 * @name ExReleasePushLock
1170 * The ExReleasePushLock macro releases a previously acquired PushLock.
1173 * Pointer to a previously acquired pushlock.
1177 * @remarks The function attempts the quickest route to release the lock, which is
1178 * to simply clear all the fields and decrease the share count if required.
1179 * However, if the pushlock is being waited on then the long path is taken.
1181 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1182 * This macro should usually be paired up with KeLeaveCriticalRegion.
1187 ExReleasePushLock(PEX_PUSH_LOCK PushLock
)
1189 EX_PUSH_LOCK OldValue
= *PushLock
;
1190 EX_PUSH_LOCK NewValue
;
1193 ASSERT(OldValue
.Locked
);
1195 /* Check if the pushlock is shared */
1196 if (OldValue
.Shared
> 1)
1198 /* Decrease the share count */
1199 NewValue
.Value
= OldValue
.Value
- EX_PUSH_LOCK_SHARE_INC
;
1203 /* Clear the pushlock entirely */
1207 /* Check if nobody is waiting on us and try clearing the lock here */
1208 if ((OldValue
.Waiting
) ||
1209 (ExpChangePushlock(PushLock
, NewValue
.Ptr
, OldValue
.Ptr
) !=
1212 /* We have waiters, use the long path */
1213 ExfReleasePushLock(PushLock
);
1217 /* FAST MUTEX INLINES *********************************************************/
1221 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex
)
1223 PKTHREAD Thread
= KeGetCurrentThread();
1226 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1227 (Thread
->CombinedApcDisable
!= 0) ||
1228 (Thread
->Teb
== NULL
) ||
1229 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1230 ASSERT(FastMutex
->Owner
!= Thread
);
1232 /* Decrease the count */
1233 if (InterlockedDecrement(&FastMutex
->Count
))
1235 /* Someone is still holding it, use slow path */
1236 KiAcquireFastMutex(FastMutex
);
1240 FastMutex
->Owner
= Thread
;
1245 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1247 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1248 (KeGetCurrentThread()->CombinedApcDisable
!= 0) ||
1249 (KeGetCurrentThread()->Teb
== NULL
) ||
1250 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1251 ASSERT(FastMutex
->Owner
== KeGetCurrentThread());
1253 /* Erase the owner */
1254 FastMutex
->Owner
= NULL
;
1256 /* Increase the count */
1257 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1259 /* Someone was waiting for it, signal the waiter */
1260 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1266 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex
)
1269 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1271 /* Raise IRQL to APC */
1272 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1274 /* Decrease the count */
1275 if (InterlockedDecrement(&FastMutex
->Count
))
1277 /* Someone is still holding it, use slow path */
1278 KiAcquireFastMutex(FastMutex
);
1281 /* Set the owner and IRQL */
1282 FastMutex
->Owner
= KeGetCurrentThread();
1283 FastMutex
->OldIrql
= OldIrql
;
1288 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1291 ASSERT(KeGetCurrentIrql() == APC_LEVEL
);
1293 /* Erase the owner */
1294 FastMutex
->Owner
= NULL
;
1295 OldIrql
= (KIRQL
)FastMutex
->OldIrql
;
1297 /* Increase the count */
1298 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1300 /* Someone was waiting for it, signal the waiter */
1301 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1304 /* Lower IRQL back */
1305 KeLowerIrql(OldIrql
);
1310 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1313 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1315 /* Raise to APC_LEVEL */
1316 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1318 /* Check if we can quickly acquire it */
1319 if (InterlockedCompareExchange(&FastMutex
->Count
, 0, 1) == 1)
1321 /* We have, set us as owners */
1322 FastMutex
->Owner
= KeGetCurrentThread();
1323 FastMutex
->OldIrql
= OldIrql
;
1328 /* Acquire attempt failed */
1329 KeLowerIrql(OldIrql
);
1337 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1339 /* Enter the Critical Region */
1340 KeEnterCriticalRegion();
1342 /* Acquire the mutex unsafely */
1343 _ExAcquireFastMutexUnsafe(FastMutex
);
1348 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex
)
1350 /* Release the mutex unsafely */
1351 _ExReleaseFastMutexUnsafe(FastMutex
);
1353 /* Leave the critical region */
1354 KeLeaveCriticalRegion();
1357 /* OTHER FUNCTIONS **********************************************************/
1361 ExTryToAcquireResourceExclusiveLite(
1362 IN PERESOURCE Resource
1366 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation
);
1370 ExAcquireTimeRefreshLock(BOOLEAN Wait
);
1374 ExReleaseTimeRefreshLock(VOID
);
1378 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime
,
1379 IN ULONG MaxSepInSeconds
);
1383 ExpAllocateLocallyUniqueId(OUT LUID
*LocallyUniqueId
);
1387 ExTimerRundown(VOID
);
1392 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1398 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1401 #define InterlockedDecrementUL(Addend) \
1402 (ULONG)InterlockedDecrement((PLONG)(Addend))
1404 #define InterlockedIncrementUL(Addend) \
1405 (ULONG)InterlockedIncrement((PLONG)(Addend))
1407 #define InterlockedExchangeUL(Target, Value) \
1408 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1410 #define InterlockedExchangeAddUL(Addend, Value) \
1411 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1413 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1414 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1416 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1417 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))