3 /* GLOBAL VARIABLES *********************************************************/
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo
;
6 extern LARGE_INTEGER ExpTimeZoneBias
;
7 extern ULONG ExpTimeZoneId
;
8 extern ULONG ExpTickCountMultiplier
;
9 extern ULONG ExpLastTimeZoneBias
;
10 extern POBJECT_TYPE ExEventPairObjectType
;
11 extern POBJECT_TYPE _ExEventObjectType
, _ExSemaphoreObjectType
;
12 extern ULONG NtBuildNumber
;
13 extern ULONG NtMajorVersion
;
14 extern ULONG NtMinorVersion
;
15 extern FAST_MUTEX ExpEnvironmentLock
;
16 extern ERESOURCE ExpFirmwareTableResource
;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead
;
18 extern BOOLEAN ExpIsWinPEMode
;
19 extern LIST_ENTRY ExpSystemResourcesList
;
20 extern ULONG ExpAnsiCodePageDataOffset
, ExpOemCodePageDataOffset
;
21 extern ULONG ExpUnicodeCaseTableDataOffset
;
22 extern PVOID ExpNlsSectionPointer
;
23 extern CHAR NtBuildLab
[];
24 extern ULONG CmNtCSDVersion
;
25 extern ULONG NtGlobalFlag
;
26 extern ULONG ExpInitializationPhase
;
27 extern ULONG ExpAltTimeZoneBias
;
28 extern LIST_ENTRY ExSystemLookasideListHead
;
30 typedef struct _EXHANDLE
39 HANDLE GenericHandleOverlay
;
42 } EXHANDLE
, *PEXHANDLE
;
44 typedef struct _ETIMER
49 LIST_ENTRY ActiveTimerListEntry
;
52 BOOLEAN ApcAssociated
;
54 LIST_ENTRY WakeTimerListEntry
;
59 PCALLBACK_OBJECT
*CallbackObject
;
63 #define MAX_FAST_REFS 7
65 #define ExAcquireRundownProtection _ExAcquireRundownProtection
66 #define ExReleaseRundownProtection _ExReleaseRundownProtection
67 #define ExInitializeRundownProtection _ExInitializeRundownProtection
68 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
69 #define ExRundownCompleted _ExRundownCompleted
70 #define ExGetPreviousMode KeGetPreviousMode
74 // Various bits tagged on the handle or handle table
76 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
77 #define FREE_HANDLE_MASK -1
80 // Number of entries in each table level
82 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
83 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
84 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
87 // Maximum index in each table level before we need another table
89 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
90 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
91 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
96 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
97 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
99 #define DEFINE_WAIT_BLOCK(x) \
103 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
105 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
106 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
111 // This is only for compatibility; the compiler will optimize the extra
112 // local variable (the actual pointer) away, so we don't take any perf hit
115 #define DEFINE_WAIT_BLOCK(x) \
116 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
117 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
122 #define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
123 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
124 #define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
126 #define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
127 #define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
128 #define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
131 /* INITIALIZATION FUNCTIONS *************************************************/
143 Phase1Initialization(
149 ExpInitializePushLocks(VOID
);
153 ExRefreshTimeZoneInformation(
154 IN PLARGE_INTEGER SystemBootTime
159 ExpInitializeWorkerThreads(VOID
);
163 ExpInitLookasideLists(VOID
);
167 ExInitializeSystemLookasideList(
168 IN PGENERAL_LOOKASIDE List
,
172 IN USHORT MaximumDepth
,
173 IN PLIST_ENTRY ListHead
178 ExpInitializeCallbacks(VOID
);
186 ExpInitializeExecutive(
188 IN PLOADER_PARAMETER_BLOCK LoaderBlock
193 ExpInitializeEventImplementation(VOID
);
197 ExpInitializeEventImplementation(VOID
);
201 ExpInitializeEventPairImplementation(VOID
);
205 ExpInitializeSemaphoreImplementation(VOID
);
209 ExpInitializeMutantImplementation(VOID
);
213 ExpInitializeTimerImplementation(VOID
);
217 ExpInitializeProfileImplementation(VOID
);
221 ExpResourceInitialization(VOID
);
225 ExInitPoolLookasidePointers(VOID
);
227 /* Callback Functions ********************************************************/
231 ExInitializeCallBack(
232 IN OUT PEX_CALLBACK Callback
235 PEX_CALLBACK_ROUTINE_BLOCK
238 IN PEX_CALLBACK_FUNCTION Function
,
245 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
250 ExCompareExchangeCallBack (
251 IN OUT PEX_CALLBACK CallBack
,
252 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock
,
253 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
256 PEX_CALLBACK_ROUTINE_BLOCK
258 ExReferenceCallBackBlock(
259 IN OUT PEX_CALLBACK CallBack
264 ExDereferenceCallBackBlock(
265 IN OUT PEX_CALLBACK CallBack
,
266 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
269 PEX_CALLBACK_FUNCTION
271 ExGetCallBackBlockRoutine(
272 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
277 ExGetCallBackBlockContext(
278 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
284 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
287 /* Rundown Functions ********************************************************/
291 ExfInitializeRundownProtection(
292 OUT PEX_RUNDOWN_REF RunRef
297 ExfReInitializeRundownProtection(
298 OUT PEX_RUNDOWN_REF RunRef
303 ExfAcquireRundownProtection(
304 IN OUT PEX_RUNDOWN_REF RunRef
309 ExfAcquireRundownProtectionEx(
310 IN OUT PEX_RUNDOWN_REF RunRef
,
316 ExfReleaseRundownProtection(
317 IN OUT PEX_RUNDOWN_REF RunRef
322 ExfReleaseRundownProtectionEx(
323 IN OUT PEX_RUNDOWN_REF RunRef
,
330 OUT PEX_RUNDOWN_REF RunRef
335 ExfWaitForRundownProtectionRelease(
336 IN OUT PEX_RUNDOWN_REF RunRef
339 /* HANDLE TABLE FUNCTIONS ***************************************************/
342 (NTAPI
*PEX_SWEEP_HANDLE_CALLBACK
)(
343 PHANDLE_TABLE_ENTRY HandleTableEntry
,
349 (NTAPI
*PEX_DUPLICATE_HANDLE_CALLBACK
)(
350 IN PEPROCESS Process
,
351 IN PHANDLE_TABLE HandleTable
,
352 IN PHANDLE_TABLE_ENTRY HandleTableEntry
,
353 IN PHANDLE_TABLE_ENTRY NewEntry
357 (NTAPI
*PEX_CHANGE_HANDLE_CALLBACK
)(
358 PHANDLE_TABLE_ENTRY HandleTableEntry
,
364 ExpInitializeHandleTables(
371 IN PEPROCESS Process OPTIONAL
376 ExUnlockHandleTableEntry(
377 IN PHANDLE_TABLE HandleTable
,
378 IN PHANDLE_TABLE_ENTRY HandleTableEntry
384 IN PHANDLE_TABLE HandleTable
,
385 IN PHANDLE_TABLE_ENTRY HandleTableEntry
390 ExDestroyHandleTable(
391 IN PHANDLE_TABLE HandleTable
,
392 IN PVOID DestroyHandleProcedure OPTIONAL
398 IN PHANDLE_TABLE HandleTable
,
400 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
405 ExMapHandleToPointer(
406 IN PHANDLE_TABLE HandleTable
,
413 IN PEPROCESS Process
,
414 IN PHANDLE_TABLE HandleTable
,
415 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure
,
422 IN PHANDLE_TABLE HandleTable
,
424 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine
,
431 IN PHANDLE_TABLE HandleTable
,
432 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure
,
436 /* PSEH EXCEPTION HANDLING **************************************************/
440 ExSystemExceptionFilter(VOID
);
442 /* CALLBACKS *****************************************************************/
446 ExDoCallBack(IN OUT PEX_CALLBACK Callback
,
451 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock
;
452 PEX_CALLBACK_FUNCTION Function
;
454 /* Reference the block */
455 CallbackBlock
= ExReferenceCallBackBlock(Callback
);
458 /* Get the function */
459 Function
= ExGetCallBackBlockRoutine(CallbackBlock
);
461 /* Do the callback */
462 Function(Context
, Argument1
, Argument2
);
464 /* Now dereference it */
465 ExDereferenceCallBackBlock(Callback
, CallbackBlock
);
469 /* FAST REFS ******************************************************************/
473 ExGetObjectFastReference(IN EX_FAST_REF FastRef
)
475 /* Return the unbiased pointer */
476 return (PVOID
)(FastRef
.Value
& ~MAX_FAST_REFS
);
481 ExGetCountFastReference(IN EX_FAST_REF FastRef
)
483 /* Return the reference count */
484 return FastRef
.RefCnt
;
489 ExInitializeFastReference(OUT PEX_FAST_REF FastRef
,
490 IN OPTIONAL PVOID Object
)
493 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
495 /* Check if an object is being set */
498 /* Clear the field */
499 FastRef
->Object
= NULL
;
503 /* Otherwise, we assume the object was referenced and is ready */
504 FastRef
->Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
510 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef
)
512 EX_FAST_REF OldValue
, NewValue
;
514 /* Start reference loop */
517 /* Get the current reference count */
521 /* Increase the reference count */
522 NewValue
.Value
= OldValue
.Value
- 1;
523 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
526 if (NewValue
.Object
!= OldValue
.Object
) continue;
533 /* Return the old value */
539 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef
,
542 EX_FAST_REF OldValue
, NewValue
;
545 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
547 /* Start update loop */
550 /* Get the current reference count */
553 /* Check if the current count is too high or if the pointer changed */
554 if (((OldValue
.RefCnt
+ MAX_FAST_REFS
) > MAX_FAST_REFS
) ||
555 ((OldValue
.Value
&~ MAX_FAST_REFS
) != (ULONG_PTR
)Object
))
561 /* Update the reference count */
562 NewValue
.Value
= OldValue
.Value
+ MAX_FAST_REFS
;
563 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
566 if (NewValue
.Object
!= OldValue
.Object
) continue;
578 ExReleaseFastReference(IN PEX_FAST_REF FastRef
,
581 EX_FAST_REF OldValue
, NewValue
;
584 ASSERT(Object
!= NULL
);
585 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
587 /* Start reference loop */
590 /* Get the current reference count */
593 /* Check if we're full if if the pointer changed */
594 if ((OldValue
.Value
^ (ULONG_PTR
)Object
) >= MAX_FAST_REFS
) return FALSE
;
596 /* Decrease the reference count */
597 NewValue
.Value
= OldValue
.Value
+ 1;
598 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
601 if (NewValue
.Object
!= OldValue
.Object
) continue;
613 ExSwapFastReference(IN PEX_FAST_REF FastRef
,
616 EX_FAST_REF NewValue
, OldValue
;
619 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
621 /* Check if an object is being set */
624 /* Clear the field */
625 NewValue
.Object
= NULL
;
629 /* Otherwise, we assume the object was referenced and is ready */
630 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
633 /* Update the object */
634 OldValue
.Object
= InterlockedExchangePointer(&FastRef
->Object
, NewValue
.Object
);
640 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef
,
644 EX_FAST_REF OldValue
, NewValue
;
646 /* Sanity check and start swap loop */
647 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
650 /* Get the current value */
653 /* Make sure there's enough references to swap */
654 if (!((OldValue
.Value
^ (ULONG_PTR
)OldObject
) <= MAX_FAST_REFS
)) break;
656 /* Check if we have an object to swap */
659 /* Set up the value with maximum fast references */
660 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
664 /* Write the object address itself (which is empty) */
665 NewValue
.Value
= (ULONG_PTR
)Object
;
668 /* Do the actual compare exchange */
669 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
672 if (NewValue
.Object
!= OldValue
.Object
) continue;
678 /* Return the old value */
682 /* RUNDOWN *******************************************************************/
685 * @name ExfAcquireRundownProtection
688 * The ExfAcquireRundownProtection routine acquires rundown protection for
689 * the specified descriptor.
692 * Pointer to a rundown reference descriptor.
694 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
696 * @remarks This is the internal macro for system use only.In case the rundown
697 * was active, then the slow-path will be called through the exported
703 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
705 ULONG_PTR Value
, NewValue
;
707 /* Get the current value and mask the active bit */
708 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
710 /* Add a reference */
711 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
713 /* Change the value */
714 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
715 if (NewValue
!= Value
)
717 /* Rundown was active, use long path */
718 return ExfAcquireRundownProtection(RunRef
);
726 * @name ExReleaseRundownProtection
729 * The ExReleaseRundownProtection routine releases rundown protection for
730 * the specified descriptor.
733 * Pointer to a rundown reference descriptor.
735 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
737 * @remarks This is the internal macro for system use only.In case the rundown
738 * was active, then the slow-path will be called through the exported
744 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
746 ULONG_PTR Value
, NewValue
;
748 /* Get the current value and mask the active bit */
749 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
751 /* Remove a reference */
752 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
754 /* Change the value */
755 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
757 /* Check if the rundown was active */
758 if (NewValue
!= Value
)
760 /* Rundown was active, use long path */
761 ExfReleaseRundownProtection(RunRef
);
766 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
771 * @name ExInitializeRundownProtection
774 * The ExInitializeRundownProtection routine initializes a rundown
775 * protection descriptor.
778 * Pointer to a rundown reference descriptor.
782 * @remarks This is the internal macro for system use only.
787 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
789 /* Set the count to zero */
794 * @name ExWaitForRundownProtectionRelease
797 * The ExWaitForRundownProtectionRelease routine waits until the specified
798 * rundown descriptor has been released.
801 * Pointer to a rundown reference descriptor.
805 * @remarks This is the internal macro for system use only. If a wait is actually
806 * necessary, then the slow path is taken through the exported function.
811 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
815 /* Set the active bit */
816 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
817 if ((Value
) && (Value
!= EX_RUNDOWN_ACTIVE
))
819 /* If the the rundown wasn't already active, then take the long path */
820 ExfWaitForRundownProtectionRelease(RunRef
);
825 * @name ExRundownCompleted
828 * The ExRundownCompleted routine completes the rundown of the specified
829 * descriptor by setting the active bit.
832 * Pointer to a rundown reference descriptor.
836 * @remarks This is the internal macro for system use only.
841 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
844 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
846 /* Mark the counter as active */
847 ExpSetRundown(&RunRef
->Count
, EX_RUNDOWN_ACTIVE
);
850 /* PUSHLOCKS *****************************************************************/
852 /* FIXME: VERIFY THESE! */
857 IN PEX_PUSH_LOCK PushLock
,
864 IN PEX_PUSH_LOCK PushLock
,
865 IN PVOID CurrentWaitBlock
870 ExWaitForUnblockPushLock(
871 IN PEX_PUSH_LOCK PushLock
,
876 * @name ExInitializePushLock
879 * The ExInitializePushLock macro initializes a PushLock.
882 * Pointer to the pushlock which is to be initialized.
891 ExInitializePushLock(IN PULONG_PTR PushLock
)
893 /* Set the value to 0 */
898 * @name ExAcquirePushLockExclusive
901 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
904 * Pointer to the pushlock which is to be acquired.
908 * @remarks The function attempts the quickest route to acquire the lock, which is
909 * to simply set the lock bit.
910 * However, if the pushlock is already shared, the slower path is taken.
912 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
913 * This macro should usually be paired up with KeAcquireCriticalRegion.
918 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
920 /* Try acquiring the lock */
921 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
923 /* Someone changed it, use the slow path */
924 ExfAcquirePushLockExclusive(PushLock
);
928 ASSERT(PushLock
->Locked
);
932 * @name ExTryToAcquirePushLockExclusive
935 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
938 * Pointer to the pushlock which is to be acquired.
942 * @remarks The function attempts the quickest route to acquire the lock, which is
943 * to simply set the lock bit.
944 * However, if the pushlock is already shared, the slower path is taken.
946 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
947 * This macro should usually be paired up with KeAcquireCriticalRegion.
952 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
954 /* Try acquiring the lock */
955 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
962 ASSERT (PushLock
->Locked
);
967 * @name ExAcquirePushLockShared
970 * The ExAcquirePushLockShared macro acquires a shared PushLock.
973 * Pointer to the pushlock which is to be acquired.
977 * @remarks The function attempts the quickest route to acquire the lock, which is
978 * to simply set the lock bit and set the share count to one.
979 * However, if the pushlock is already shared, the slower path is taken.
981 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
982 * This macro should usually be paired up with KeAcquireCriticalRegion.
987 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
989 EX_PUSH_LOCK NewValue
;
991 /* Try acquiring the lock */
992 NewValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
993 if (ExpChangePushlock(PushLock
, NewValue
.Ptr
, 0))
995 /* Someone changed it, use the slow path */
996 ExfAcquirePushLockShared(PushLock
);
1000 ASSERT(PushLock
->Locked
);
1001 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1005 * @name ExConvertPushLockSharedToExclusive
1008 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1009 * pushlock to a shared pushlock.
1012 * Pointer to the pushlock which is to be converted.
1014 * @return FALSE if conversion failed, TRUE otherwise.
1016 * @remarks The function attempts the quickest route to convert the lock, which is
1017 * to simply set the lock bit and remove any other bits.
1022 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock
)
1024 EX_PUSH_LOCK OldValue
;
1026 /* Set the expected old value */
1027 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1029 /* Try converting the lock */
1030 if (ExpChangePushlock(PushLock
, EX_PUSH_LOCK_LOCK
, OldValue
.Value
) !=
1033 /* Conversion failed */
1038 ASSERT(PushLock
->Locked
);
1043 * @name ExWaitOnPushLock
1046 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1049 * Pointer to a pushlock.
1053 * @remarks The function attempts to get any exclusive waiters out of their slow
1054 * path by forcing an instant acquire/release operation.
1056 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1061 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock
)
1063 /* Check if we're locked */
1064 if (PushLock
->Locked
)
1066 /* Acquire the lock */
1067 ExfAcquirePushLockExclusive(PushLock
);
1068 ASSERT(PushLock
->Locked
);
1071 ExfReleasePushLockExclusive(PushLock
);
1076 * @name ExReleasePushLockShared
1079 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1082 * Pointer to a previously acquired pushlock.
1086 * @remarks The function attempts the quickest route to release the lock, which is
1087 * to simply decrease the share count and remove the lock bit.
1088 * However, if the pushlock is being waited on then the long path is taken.
1090 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1091 * This macro should usually be paired up with KeLeaveCriticalRegion.
1096 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
1098 EX_PUSH_LOCK OldValue
;
1101 ASSERT(PushLock
->Locked
);
1102 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1104 /* Try to clear the pushlock */
1105 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1106 if (ExpChangePushlock(PushLock
, 0, OldValue
.Ptr
) != OldValue
.Ptr
)
1108 /* There are still other people waiting on it */
1109 ExfReleasePushLockShared(PushLock
);
1114 * @name ExReleasePushLockExclusive
1117 * The ExReleasePushLockExclusive macro releases a previously
1118 * exclusively acquired PushLock.
1121 * Pointer to a previously acquired pushlock.
1125 * @remarks The function attempts the quickest route to release the lock, which is
1126 * to simply clear the locked bit.
1127 * However, if the pushlock is being waited on, the slow path is taken
1128 * in an attempt to wake up the lock.
1130 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1131 * This macro should usually be paired up with KeLeaveCriticalRegion.
1136 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1138 EX_PUSH_LOCK OldValue
;
1141 ASSERT(PushLock
->Locked
);
1142 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
== 0);
1144 /* Unlock the pushlock */
1145 OldValue
.Value
= InterlockedExchangeAddSizeT((PSIZE_T
)PushLock
,
1146 -(SIZE_T
)EX_PUSH_LOCK_LOCK
);
1149 ASSERT(OldValue
.Locked
);
1150 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1152 /* Check if anyone is waiting on it and it's not already waking*/
1153 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1156 ExfTryToWakePushLock(PushLock
);
1161 * @name ExReleasePushLock
1164 * The ExReleasePushLock macro releases a previously acquired PushLock.
1167 * Pointer to a previously acquired pushlock.
1171 * @remarks The function attempts the quickest route to release the lock, which is
1172 * to simply clear all the fields and decrease the share count if required.
1173 * However, if the pushlock is being waited on then the long path is taken.
1175 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1176 * This macro should usually be paired up with KeLeaveCriticalRegion.
1181 ExReleasePushLock(PEX_PUSH_LOCK PushLock
)
1183 EX_PUSH_LOCK OldValue
= *PushLock
;
1184 EX_PUSH_LOCK NewValue
;
1187 ASSERT(OldValue
.Locked
);
1189 /* Check if the pushlock is shared */
1190 if (OldValue
.Shared
> 1)
1192 /* Decrease the share count */
1193 NewValue
.Value
= OldValue
.Value
- EX_PUSH_LOCK_SHARE_INC
;
1197 /* Clear the pushlock entirely */
1201 /* Check if nobody is waiting on us and try clearing the lock here */
1202 if ((OldValue
.Waiting
) ||
1203 (ExpChangePushlock(PushLock
, NewValue
.Ptr
, OldValue
.Ptr
) !=
1206 /* We have waiters, use the long path */
1207 ExfReleasePushLock(PushLock
);
1211 /* FAST MUTEX INLINES *********************************************************/
1215 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex
)
1217 PKTHREAD Thread
= KeGetCurrentThread();
1220 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1221 (Thread
->CombinedApcDisable
!= 0) ||
1222 (Thread
->Teb
== NULL
) ||
1223 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1224 ASSERT(FastMutex
->Owner
!= Thread
);
1226 /* Decrease the count */
1227 if (InterlockedDecrement(&FastMutex
->Count
))
1229 /* Someone is still holding it, use slow path */
1230 KiAcquireFastMutex(FastMutex
);
1234 FastMutex
->Owner
= Thread
;
1239 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1241 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1242 (KeGetCurrentThread()->CombinedApcDisable
!= 0) ||
1243 (KeGetCurrentThread()->Teb
== NULL
) ||
1244 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1245 ASSERT(FastMutex
->Owner
== KeGetCurrentThread());
1247 /* Erase the owner */
1248 FastMutex
->Owner
= NULL
;
1250 /* Increase the count */
1251 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1253 /* Someone was waiting for it, signal the waiter */
1254 KeSetEventBoostPriority(&FastMutex
->Gate
, NULL
);
1260 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex
)
1263 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1265 /* Raise IRQL to APC */
1266 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1268 /* Decrease the count */
1269 if (InterlockedDecrement(&FastMutex
->Count
))
1271 /* Someone is still holding it, use slow path */
1272 KiAcquireFastMutex(FastMutex
);
1275 /* Set the owner and IRQL */
1276 FastMutex
->Owner
= KeGetCurrentThread();
1277 FastMutex
->OldIrql
= OldIrql
;
1282 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1285 ASSERT(KeGetCurrentIrql() == APC_LEVEL
);
1287 /* Erase the owner */
1288 FastMutex
->Owner
= NULL
;
1289 OldIrql
= (KIRQL
)FastMutex
->OldIrql
;
1291 /* Increase the count */
1292 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1294 /* Someone was waiting for it, signal the waiter */
1295 KeSetEventBoostPriority(&FastMutex
->Gate
, NULL
);
1298 /* Lower IRQL back */
1299 KeLowerIrql(OldIrql
);
1304 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1307 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1309 /* Raise to APC_LEVEL */
1310 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1312 /* Check if we can quickly acquire it */
1313 if (InterlockedCompareExchange(&FastMutex
->Count
, 0, 1) == 1)
1315 /* We have, set us as owners */
1316 FastMutex
->Owner
= KeGetCurrentThread();
1317 FastMutex
->OldIrql
= OldIrql
;
1322 /* Acquire attempt failed */
1323 KeLowerIrql(OldIrql
);
1331 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1333 /* Enter the Critical Region */
1334 KeEnterCriticalRegion();
1336 /* Acquire the mutex unsafely */
1337 _ExAcquireFastMutexUnsafe(FastMutex
);
1342 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex
)
1344 /* Release the mutex unsafely */
1345 _ExReleaseFastMutexUnsafe(FastMutex
);
1347 /* Leave the critical region */
1348 KeLeaveCriticalRegion();
1351 /* OTHER FUNCTIONS **********************************************************/
1355 ExTryToAcquireResourceExclusiveLite(
1356 IN PERESOURCE Resource
1360 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation
);
1364 ExAcquireTimeRefreshLock(BOOLEAN Wait
);
1368 ExReleaseTimeRefreshLock(VOID
);
1372 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime
,
1373 IN ULONG MaxSepInSeconds
);
1377 ExpAllocateLocallyUniqueId(OUT LUID
*LocallyUniqueId
);
1381 ExTimerRundown(VOID
);
1386 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1392 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1395 #define InterlockedDecrementUL(Addend) \
1396 (ULONG)InterlockedDecrement((PLONG)(Addend))
1398 #define InterlockedIncrementUL(Addend) \
1399 (ULONG)InterlockedIncrement((PLONG)(Addend))
1401 #define InterlockedExchangeUL(Target, Value) \
1402 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1404 #define InterlockedExchangeAddUL(Addend, Value) \
1405 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1407 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1408 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1410 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1411 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))