3 /* GLOBAL VARIABLES *********************************************************/
5 extern TIME_ZONE_INFORMATION ExpTimeZoneInfo
;
6 extern LARGE_INTEGER ExpTimeZoneBias
;
7 extern ULONG ExpTimeZoneId
;
8 extern ULONG ExpTickCountMultiplier
;
9 extern ULONG ExpLastTimeZoneBias
;
10 extern POBJECT_TYPE ExEventPairObjectType
;
11 extern POBJECT_TYPE _ExEventObjectType
, _ExSemaphoreObjectType
;
12 extern ULONG NtBuildNumber
;
13 extern ULONG NtMajorVersion
;
14 extern ULONG NtMinorVersion
;
15 extern FAST_MUTEX ExpEnvironmentLock
;
16 extern ERESOURCE ExpFirmwareTableResource
;
17 extern LIST_ENTRY ExpFirmwareTableProviderListHead
;
18 extern BOOLEAN ExpIsWinPEMode
;
19 extern LIST_ENTRY ExpSystemResourcesList
;
20 extern ULONG ExpAnsiCodePageDataOffset
, ExpOemCodePageDataOffset
;
21 extern ULONG ExpUnicodeCaseTableDataOffset
;
22 extern PVOID ExpNlsSectionPointer
;
23 extern CHAR NtBuildLab
[];
24 extern ULONG CmNtCSDVersion
;
25 extern ULONG NtGlobalFlag
;
26 extern ULONG ExpInitializationPhase
;
27 extern ULONG ExpAltTimeZoneBias
;
28 extern LIST_ENTRY ExSystemLookasideListHead
;
29 extern PCALLBACK_OBJECT PowerStateCallback
;
31 typedef struct _EXHANDLE
40 HANDLE GenericHandleOverlay
;
43 } EXHANDLE
, *PEXHANDLE
;
45 typedef struct _ETIMER
50 LIST_ENTRY ActiveTimerListEntry
;
53 BOOLEAN ApcAssociated
;
55 LIST_ENTRY WakeTimerListEntry
;
60 PCALLBACK_OBJECT
*CallbackObject
;
64 typedef struct _HARDERROR_USER_PARAMETERS
66 ULONG_PTR Parameters
[MAXIMUM_HARDERROR_PARAMETERS
];
67 UNICODE_STRING Strings
[MAXIMUM_HARDERROR_PARAMETERS
];
68 WCHAR Buffer
[ANYSIZE_ARRAY
];
69 } HARDERROR_USER_PARAMETERS
, *PHARDERROR_USER_PARAMETERS
;
71 #define MAX_FAST_REFS 7
73 #define ExAcquireRundownProtection _ExAcquireRundownProtection
74 #define ExReleaseRundownProtection _ExReleaseRundownProtection
75 #define ExInitializeRundownProtection _ExInitializeRundownProtection
76 #define ExWaitForRundownProtectionRelease _ExWaitForRundownProtectionRelease
77 #define ExRundownCompleted _ExRundownCompleted
78 #define ExGetPreviousMode KeGetPreviousMode
82 // Various bits tagged on the handle or handle table
84 #define EXHANDLE_TABLE_ENTRY_LOCK_BIT 1
85 #define FREE_HANDLE_MASK -1
88 // Number of entries in each table level
90 #define LOW_LEVEL_ENTRIES (PAGE_SIZE / sizeof(HANDLE_TABLE_ENTRY))
91 #define MID_LEVEL_ENTRIES (PAGE_SIZE / sizeof(PHANDLE_TABLE_ENTRY))
92 #define HIGH_LEVEL_ENTRIES (16777216 / (LOW_LEVEL_ENTRIES * MID_LEVEL_ENTRIES))
95 // Maximum index in each table level before we need another table
97 #define MAX_LOW_INDEX LOW_LEVEL_ENTRIES
98 #define MAX_MID_INDEX (MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
99 #define MAX_HIGH_INDEX (MID_LEVEL_ENTRIES * MID_LEVEL_ENTRIES * LOW_LEVEL_ENTRIES)
104 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
105 (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
107 #define DEFINE_WAIT_BLOCK(x) \
111 EX_PUSH_LOCK_WAIT_BLOCK UnalignedBlock; \
113 PEX_PUSH_LOCK_WAIT_BLOCK x = (PEX_PUSH_LOCK_WAIT_BLOCK) \
114 ((ULONG_PTR)&WaitBlockBuffer.UnalignedBlock &~ 0xF);
119 // This is only for compatibility; the compiler will optimize the extra
120 // local variable (the actual pointer) away, so we don't take any perf hit
123 #define DEFINE_WAIT_BLOCK(x) \
124 EX_PUSH_LOCK_WAIT_BLOCK WaitBlockBuffer; \
125 PEX_PUSH_LOCK_WAIT_BLOCK x = &WaitBlockBuffer;
129 #define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
130 #define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
131 #define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
136 IN PSYSTEM_POOLTAG_INFORMATION SystemInformation
,
137 IN ULONG SystemInformationLength
,
138 IN OUT PULONG ReturnLength OPTIONAL
141 /* INITIALIZATION FUNCTIONS *************************************************/
153 Phase1Initialization(
159 ExpInitializePushLocks(VOID
);
163 ExRefreshTimeZoneInformation(
164 IN PLARGE_INTEGER SystemBootTime
169 ExpInitializeWorkerThreads(VOID
);
173 ExSwapinWorkerThreads(IN BOOLEAN AllowSwap
);
177 ExpInitLookasideLists(VOID
);
181 ExInitializeSystemLookasideList(
182 IN PGENERAL_LOOKASIDE List
,
186 IN USHORT MaximumDepth
,
187 IN PLIST_ENTRY ListHead
192 ExpInitializeCallbacks(VOID
);
200 ExpInitializeExecutive(
202 IN PLOADER_PARAMETER_BLOCK LoaderBlock
207 ExpInitializeEventImplementation(VOID
);
211 ExpInitializeKeyedEventImplementation(VOID
);
215 ExpInitializeEventPairImplementation(VOID
);
219 ExpInitializeSemaphoreImplementation(VOID
);
223 ExpInitializeMutantImplementation(VOID
);
227 ExpInitializeTimerImplementation(VOID
);
231 ExpInitializeProfileImplementation(VOID
);
235 ExpResourceInitialization(VOID
);
239 ExInitPoolLookasidePointers(VOID
);
241 /* Callback Functions ********************************************************/
245 ExInitializeCallBack(
246 IN OUT PEX_CALLBACK Callback
249 PEX_CALLBACK_ROUTINE_BLOCK
252 IN PEX_CALLBACK_FUNCTION Function
,
259 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
264 ExCompareExchangeCallBack (
265 IN OUT PEX_CALLBACK CallBack
,
266 IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock
,
267 IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock
270 PEX_CALLBACK_ROUTINE_BLOCK
272 ExReferenceCallBackBlock(
273 IN OUT PEX_CALLBACK CallBack
278 ExDereferenceCallBackBlock(
279 IN OUT PEX_CALLBACK CallBack
,
280 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
283 PEX_CALLBACK_FUNCTION
285 ExGetCallBackBlockRoutine(
286 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
291 ExGetCallBackBlockContext(
292 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
298 IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock
301 /* Rundown Functions ********************************************************/
305 ExfInitializeRundownProtection(
306 OUT PEX_RUNDOWN_REF RunRef
311 ExfReInitializeRundownProtection(
312 OUT PEX_RUNDOWN_REF RunRef
317 ExfAcquireRundownProtection(
318 IN OUT PEX_RUNDOWN_REF RunRef
323 ExfAcquireRundownProtectionEx(
324 IN OUT PEX_RUNDOWN_REF RunRef
,
330 ExfReleaseRundownProtection(
331 IN OUT PEX_RUNDOWN_REF RunRef
336 ExfReleaseRundownProtectionEx(
337 IN OUT PEX_RUNDOWN_REF RunRef
,
344 OUT PEX_RUNDOWN_REF RunRef
349 ExfWaitForRundownProtectionRelease(
350 IN OUT PEX_RUNDOWN_REF RunRef
353 /* HANDLE TABLE FUNCTIONS ***************************************************/
356 (NTAPI
*PEX_SWEEP_HANDLE_CALLBACK
)(
357 PHANDLE_TABLE_ENTRY HandleTableEntry
,
363 (NTAPI
*PEX_DUPLICATE_HANDLE_CALLBACK
)(
364 IN PEPROCESS Process
,
365 IN PHANDLE_TABLE HandleTable
,
366 IN PHANDLE_TABLE_ENTRY HandleTableEntry
,
367 IN PHANDLE_TABLE_ENTRY NewEntry
371 (NTAPI
*PEX_CHANGE_HANDLE_CALLBACK
)(
372 PHANDLE_TABLE_ENTRY HandleTableEntry
,
378 ExpInitializeHandleTables(
385 IN PEPROCESS Process OPTIONAL
390 ExUnlockHandleTableEntry(
391 IN PHANDLE_TABLE HandleTable
,
392 IN PHANDLE_TABLE_ENTRY HandleTableEntry
398 IN PHANDLE_TABLE HandleTable
,
399 IN PHANDLE_TABLE_ENTRY HandleTableEntry
404 ExDestroyHandleTable(
405 IN PHANDLE_TABLE HandleTable
,
406 IN PVOID DestroyHandleProcedure OPTIONAL
412 IN PHANDLE_TABLE HandleTable
,
414 IN PHANDLE_TABLE_ENTRY HandleTableEntry OPTIONAL
419 ExMapHandleToPointer(
420 IN PHANDLE_TABLE HandleTable
,
427 IN PEPROCESS Process
,
428 IN PHANDLE_TABLE HandleTable
,
429 IN PEX_DUPLICATE_HANDLE_CALLBACK DupHandleProcedure
,
436 IN PHANDLE_TABLE HandleTable
,
438 IN PEX_CHANGE_HANDLE_CALLBACK ChangeRoutine
,
445 IN PHANDLE_TABLE HandleTable
,
446 IN PEX_SWEEP_HANDLE_CALLBACK EnumHandleProcedure
,
450 /* PSEH EXCEPTION HANDLING **************************************************/
454 ExSystemExceptionFilter(VOID
);
456 /* CALLBACKS *****************************************************************/
460 ExDoCallBack(IN OUT PEX_CALLBACK Callback
,
465 PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock
;
466 PEX_CALLBACK_FUNCTION Function
;
468 /* Reference the block */
469 CallbackBlock
= ExReferenceCallBackBlock(Callback
);
472 /* Get the function */
473 Function
= ExGetCallBackBlockRoutine(CallbackBlock
);
475 /* Do the callback */
476 Function(Context
, Argument1
, Argument2
);
478 /* Now dereference it */
479 ExDereferenceCallBackBlock(Callback
, CallbackBlock
);
483 /* FAST REFS ******************************************************************/
487 ExGetObjectFastReference(IN EX_FAST_REF FastRef
)
489 /* Return the unbiased pointer */
490 return (PVOID
)(FastRef
.Value
& ~MAX_FAST_REFS
);
495 ExGetCountFastReference(IN EX_FAST_REF FastRef
)
497 /* Return the reference count */
498 return (ULONG
)FastRef
.RefCnt
;
503 ExInitializeFastReference(OUT PEX_FAST_REF FastRef
,
504 IN OPTIONAL PVOID Object
)
507 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
509 /* Check if an object is being set */
512 /* Clear the field */
513 FastRef
->Object
= NULL
;
517 /* Otherwise, we assume the object was referenced and is ready */
518 FastRef
->Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
524 ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef
)
526 EX_FAST_REF OldValue
, NewValue
;
528 /* Start reference loop */
531 /* Get the current reference count */
535 /* Increase the reference count */
536 NewValue
.Value
= OldValue
.Value
- 1;
537 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
540 if (NewValue
.Object
!= OldValue
.Object
) continue;
547 /* Return the old value */
553 ExInsertFastReference(IN OUT PEX_FAST_REF FastRef
,
556 EX_FAST_REF OldValue
, NewValue
;
559 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
561 /* Start update loop */
564 /* Get the current reference count */
567 /* Check if the current count is too high or if the pointer changed */
568 if (((OldValue
.RefCnt
+ MAX_FAST_REFS
) > MAX_FAST_REFS
) ||
569 ((OldValue
.Value
&~ MAX_FAST_REFS
) != (ULONG_PTR
)Object
))
575 /* Update the reference count */
576 NewValue
.Value
= OldValue
.Value
+ MAX_FAST_REFS
;
577 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
580 if (NewValue
.Object
!= OldValue
.Object
) continue;
592 ExReleaseFastReference(IN PEX_FAST_REF FastRef
,
595 EX_FAST_REF OldValue
, NewValue
;
598 ASSERT(Object
!= NULL
);
599 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
601 /* Start reference loop */
604 /* Get the current reference count */
607 /* Check if we're full if if the pointer changed */
608 if ((OldValue
.Value
^ (ULONG_PTR
)Object
) >= MAX_FAST_REFS
) return FALSE
;
610 /* Decrease the reference count */
611 NewValue
.Value
= OldValue
.Value
+ 1;
612 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
615 if (NewValue
.Object
!= OldValue
.Object
) continue;
627 ExSwapFastReference(IN PEX_FAST_REF FastRef
,
630 EX_FAST_REF NewValue
, OldValue
;
633 ASSERT((((ULONG_PTR
)Object
) & MAX_FAST_REFS
) == 0);
635 /* Check if an object is being set */
638 /* Clear the field */
639 NewValue
.Object
= NULL
;
643 /* Otherwise, we assume the object was referenced and is ready */
644 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
647 /* Update the object */
648 OldValue
.Object
= InterlockedExchangePointer(&FastRef
->Object
, NewValue
.Object
);
654 ExCompareSwapFastReference(IN PEX_FAST_REF FastRef
,
658 EX_FAST_REF OldValue
, NewValue
;
660 /* Sanity check and start swap loop */
661 ASSERT(!(((ULONG_PTR
)Object
) & MAX_FAST_REFS
));
664 /* Get the current value */
667 /* Make sure there's enough references to swap */
668 if (!((OldValue
.Value
^ (ULONG_PTR
)OldObject
) <= MAX_FAST_REFS
)) break;
670 /* Check if we have an object to swap */
673 /* Set up the value with maximum fast references */
674 NewValue
.Value
= (ULONG_PTR
)Object
| MAX_FAST_REFS
;
678 /* Write the object address itself (which is empty) */
679 NewValue
.Value
= (ULONG_PTR
)Object
;
682 /* Do the actual compare exchange */
683 NewValue
.Object
= ExpChangePushlock(&FastRef
->Object
,
686 if (NewValue
.Object
!= OldValue
.Object
) continue;
692 /* Return the old value */
696 /* RUNDOWN *******************************************************************/
699 * @name ExfAcquireRundownProtection
702 * The ExfAcquireRundownProtection routine acquires rundown protection for
703 * the specified descriptor.
706 * Pointer to a rundown reference descriptor.
708 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
710 * @remarks This is the internal macro for system use only.In case the rundown
711 * was active, then the slow-path will be called through the exported
717 _ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
719 ULONG_PTR Value
, NewValue
;
721 /* Get the current value and mask the active bit */
722 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
724 /* Add a reference */
725 NewValue
= Value
+ EX_RUNDOWN_COUNT_INC
;
727 /* Change the value */
728 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
729 if (NewValue
!= Value
)
731 /* Rundown was active, use long path */
732 return ExfAcquireRundownProtection(RunRef
);
740 * @name ExReleaseRundownProtection
743 * The ExReleaseRundownProtection routine releases rundown protection for
744 * the specified descriptor.
747 * Pointer to a rundown reference descriptor.
749 * @return TRUE if access to the protected structure was granted, FALSE otherwise.
751 * @remarks This is the internal macro for system use only.In case the rundown
752 * was active, then the slow-path will be called through the exported
758 _ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
760 ULONG_PTR Value
, NewValue
;
762 /* Get the current value and mask the active bit */
763 Value
= RunRef
->Count
&~ EX_RUNDOWN_ACTIVE
;
765 /* Remove a reference */
766 NewValue
= Value
- EX_RUNDOWN_COUNT_INC
;
768 /* Change the value */
769 NewValue
= ExpChangeRundown(RunRef
, NewValue
, Value
);
771 /* Check if the rundown was active */
772 if (NewValue
!= Value
)
774 /* Rundown was active, use long path */
775 ExfReleaseRundownProtection(RunRef
);
780 ASSERT((Value
>= EX_RUNDOWN_COUNT_INC
) || (KeNumberProcessors
> 1));
785 * @name ExInitializeRundownProtection
788 * The ExInitializeRundownProtection routine initializes a rundown
789 * protection descriptor.
792 * Pointer to a rundown reference descriptor.
796 * @remarks This is the internal macro for system use only.
801 _ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef
)
803 /* Set the count to zero */
808 * @name ExWaitForRundownProtectionRelease
811 * The ExWaitForRundownProtectionRelease routine waits until the specified
812 * rundown descriptor has been released.
815 * Pointer to a rundown reference descriptor.
819 * @remarks This is the internal macro for system use only. If a wait is actually
820 * necessary, then the slow path is taken through the exported function.
825 _ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef
)
829 /* Set the active bit */
830 Value
= ExpChangeRundown(RunRef
, EX_RUNDOWN_ACTIVE
, 0);
831 if ((Value
) && (Value
!= EX_RUNDOWN_ACTIVE
))
833 /* If the the rundown wasn't already active, then take the long path */
834 ExfWaitForRundownProtectionRelease(RunRef
);
839 * @name ExRundownCompleted
842 * The ExRundownCompleted routine completes the rundown of the specified
843 * descriptor by setting the active bit.
846 * Pointer to a rundown reference descriptor.
850 * @remarks This is the internal macro for system use only.
855 _ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef
)
858 ASSERT((RunRef
->Count
& EX_RUNDOWN_ACTIVE
) != 0);
860 /* Mark the counter as active */
861 ExpSetRundown(RunRef
, EX_RUNDOWN_ACTIVE
);
864 /* PUSHLOCKS *****************************************************************/
866 /* FIXME: VERIFY THESE! */
871 IN PEX_PUSH_LOCK PushLock
,
878 IN PEX_PUSH_LOCK PushLock
,
879 IN PVOID CurrentWaitBlock
884 ExWaitForUnblockPushLock(
885 IN PEX_PUSH_LOCK PushLock
,
890 * @name _ExInitializePushLock
893 * The _ExInitializePushLock macro initializes a PushLock.
896 * Pointer to the pushlock which is to be initialized.
905 _ExInitializePushLock(OUT PEX_PUSH_LOCK PushLock
)
907 /* Set the value to 0 */
910 #define ExInitializePushLock _ExInitializePushLock
913 * @name ExAcquirePushLockExclusive
916 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
919 * Pointer to the pushlock which is to be acquired.
923 * @remarks The function attempts the quickest route to acquire the lock, which is
924 * to simply set the lock bit.
925 * However, if the pushlock is already shared, the slower path is taken.
927 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
928 * This macro should usually be paired up with KeAcquireCriticalRegion.
933 ExAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
935 /* Try acquiring the lock */
936 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
938 /* Someone changed it, use the slow path */
939 ExfAcquirePushLockExclusive(PushLock
);
943 ASSERT(PushLock
->Locked
);
947 * @name ExTryToAcquirePushLockExclusive
950 * The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
953 * Pointer to the pushlock which is to be acquired.
957 * @remarks The function attempts the quickest route to acquire the lock, which is
958 * to simply set the lock bit.
959 * However, if the pushlock is already shared, the slower path is taken.
961 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
962 * This macro should usually be paired up with KeAcquireCriticalRegion.
967 ExTryToAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock
)
969 /* Try acquiring the lock */
970 if (InterlockedBitTestAndSet((PLONG
)PushLock
, EX_PUSH_LOCK_LOCK_V
))
977 ASSERT (PushLock
->Locked
);
982 * @name ExAcquirePushLockShared
985 * The ExAcquirePushLockShared macro acquires a shared PushLock.
988 * Pointer to the pushlock which is to be acquired.
992 * @remarks The function attempts the quickest route to acquire the lock, which is
993 * to simply set the lock bit and set the share count to one.
994 * However, if the pushlock is already shared, the slower path is taken.
996 * Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
997 * This macro should usually be paired up with KeAcquireCriticalRegion.
1002 ExAcquirePushLockShared(PEX_PUSH_LOCK PushLock
)
1004 EX_PUSH_LOCK NewValue
;
1006 /* Try acquiring the lock */
1007 NewValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1008 if (ExpChangePushlock(PushLock
, NewValue
.Ptr
, 0))
1010 /* Someone changed it, use the slow path */
1011 ExfAcquirePushLockShared(PushLock
);
1015 ASSERT(PushLock
->Locked
);
1016 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1020 * @name ExConvertPushLockSharedToExclusive
1023 * The ExConvertPushLockSharedToExclusive macro converts an exclusive
1024 * pushlock to a shared pushlock.
1027 * Pointer to the pushlock which is to be converted.
1029 * @return FALSE if conversion failed, TRUE otherwise.
1031 * @remarks The function attempts the quickest route to convert the lock, which is
1032 * to simply set the lock bit and remove any other bits.
1037 ExConvertPushLockSharedToExclusive(IN PEX_PUSH_LOCK PushLock
)
1039 EX_PUSH_LOCK OldValue
;
1041 /* Set the expected old value */
1042 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1044 /* Try converting the lock */
1045 if (ExpChangePushlock(PushLock
, EX_PUSH_LOCK_LOCK
, OldValue
.Value
) !=
1048 /* Conversion failed */
1053 ASSERT(PushLock
->Locked
);
1058 * @name ExWaitOnPushLock
1061 * The ExWaitOnPushLock macro acquires and instantly releases a pushlock.
1064 * Pointer to a pushlock.
1068 * @remarks The function attempts to get any exclusive waiters out of their slow
1069 * path by forcing an instant acquire/release operation.
1071 * Callers of ExWaitOnPushLock must be running at IRQL <= APC_LEVEL.
1076 ExWaitOnPushLock(PEX_PUSH_LOCK PushLock
)
1078 /* Check if we're locked */
1079 if (PushLock
->Locked
)
1081 /* Acquire the lock */
1082 ExfAcquirePushLockExclusive(PushLock
);
1083 ASSERT(PushLock
->Locked
);
1086 ExfReleasePushLockExclusive(PushLock
);
1091 * @name ExReleasePushLockShared
1094 * The ExReleasePushLockShared macro releases a previously acquired PushLock.
1097 * Pointer to a previously acquired pushlock.
1101 * @remarks The function attempts the quickest route to release the lock, which is
1102 * to simply decrease the share count and remove the lock bit.
1103 * However, if the pushlock is being waited on then the long path is taken.
1105 * Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
1106 * This macro should usually be paired up with KeLeaveCriticalRegion.
1111 ExReleasePushLockShared(PEX_PUSH_LOCK PushLock
)
1113 EX_PUSH_LOCK OldValue
;
1116 ASSERT(PushLock
->Locked
);
1117 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
> 0);
1119 /* Try to clear the pushlock */
1120 OldValue
.Value
= EX_PUSH_LOCK_LOCK
| EX_PUSH_LOCK_SHARE_INC
;
1121 if (ExpChangePushlock(PushLock
, 0, OldValue
.Ptr
) != OldValue
.Ptr
)
1123 /* There are still other people waiting on it */
1124 ExfReleasePushLockShared(PushLock
);
1129 * @name ExReleasePushLockExclusive
1132 * The ExReleasePushLockExclusive macro releases a previously
1133 * exclusively acquired PushLock.
1136 * Pointer to a previously acquired pushlock.
1140 * @remarks The function attempts the quickest route to release the lock, which is
1141 * to simply clear the locked bit.
1142 * However, if the pushlock is being waited on, the slow path is taken
1143 * in an attempt to wake up the lock.
1145 * Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
1146 * This macro should usually be paired up with KeLeaveCriticalRegion.
1151 ExReleasePushLockExclusive(PEX_PUSH_LOCK PushLock
)
1153 EX_PUSH_LOCK OldValue
;
1156 ASSERT(PushLock
->Locked
);
1157 ASSERT(PushLock
->Waiting
|| PushLock
->Shared
== 0);
1159 /* Unlock the pushlock */
1160 OldValue
.Value
= InterlockedExchangeAddSizeT((PSIZE_T
)PushLock
,
1161 -(SSIZE_T
)EX_PUSH_LOCK_LOCK
);
1164 ASSERT(OldValue
.Locked
);
1165 ASSERT(OldValue
.Waiting
|| OldValue
.Shared
== 0);
1167 /* Check if anyone is waiting on it and it's not already waking*/
1168 if ((OldValue
.Waiting
) && !(OldValue
.Waking
))
1171 ExfTryToWakePushLock(PushLock
);
1176 * @name ExReleasePushLock
1179 * The ExReleasePushLock macro releases a previously acquired PushLock.
1182 * Pointer to a previously acquired pushlock.
1186 * @remarks The function attempts the quickest route to release the lock, which is
1187 * to simply clear all the fields and decrease the share count if required.
1188 * However, if the pushlock is being waited on then the long path is taken.
1190 * Callers of ExReleasePushLock must be running at IRQL <= APC_LEVEL.
1191 * This macro should usually be paired up with KeLeaveCriticalRegion.
1196 ExReleasePushLock(PEX_PUSH_LOCK PushLock
)
1198 EX_PUSH_LOCK OldValue
= *PushLock
;
1199 EX_PUSH_LOCK NewValue
;
1202 ASSERT(OldValue
.Locked
);
1204 /* Check if the pushlock is shared */
1205 if (OldValue
.Shared
> 1)
1207 /* Decrease the share count */
1208 NewValue
.Value
= OldValue
.Value
- EX_PUSH_LOCK_SHARE_INC
;
1212 /* Clear the pushlock entirely */
1216 /* Check if nobody is waiting on us and try clearing the lock here */
1217 if ((OldValue
.Waiting
) ||
1218 (ExpChangePushlock(PushLock
, NewValue
.Ptr
, OldValue
.Ptr
) !=
1221 /* We have waiters, use the long path */
1222 ExfReleasePushLock(PushLock
);
1226 /* FAST MUTEX INLINES *********************************************************/
1230 _ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex
)
1232 PKTHREAD Thread
= KeGetCurrentThread();
1235 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1236 (Thread
->CombinedApcDisable
!= 0) ||
1237 (Thread
->Teb
== NULL
) ||
1238 (Thread
->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1239 ASSERT(FastMutex
->Owner
!= Thread
);
1241 /* Decrease the count */
1242 if (InterlockedDecrement(&FastMutex
->Count
))
1244 /* Someone is still holding it, use slow path */
1245 KiAcquireFastMutex(FastMutex
);
1249 FastMutex
->Owner
= Thread
;
1254 _ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1256 ASSERT((KeGetCurrentIrql() == APC_LEVEL
) ||
1257 (KeGetCurrentThread()->CombinedApcDisable
!= 0) ||
1258 (KeGetCurrentThread()->Teb
== NULL
) ||
1259 (KeGetCurrentThread()->Teb
>= (PTEB
)MM_SYSTEM_RANGE_START
));
1260 ASSERT(FastMutex
->Owner
== KeGetCurrentThread());
1262 /* Erase the owner */
1263 FastMutex
->Owner
= NULL
;
1265 /* Increase the count */
1266 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1268 /* Someone was waiting for it, signal the waiter */
1269 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1275 _ExAcquireFastMutex(IN PFAST_MUTEX FastMutex
)
1278 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1280 /* Raise IRQL to APC */
1281 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1283 /* Decrease the count */
1284 if (InterlockedDecrement(&FastMutex
->Count
))
1286 /* Someone is still holding it, use slow path */
1287 KiAcquireFastMutex(FastMutex
);
1290 /* Set the owner and IRQL */
1291 FastMutex
->Owner
= KeGetCurrentThread();
1292 FastMutex
->OldIrql
= OldIrql
;
1297 _ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1300 ASSERT(KeGetCurrentIrql() == APC_LEVEL
);
1302 /* Erase the owner */
1303 FastMutex
->Owner
= NULL
;
1304 OldIrql
= (KIRQL
)FastMutex
->OldIrql
;
1306 /* Increase the count */
1307 if (InterlockedIncrement(&FastMutex
->Count
) <= 0)
1309 /* Someone was waiting for it, signal the waiter */
1310 KeSetEventBoostPriority(&FastMutex
->Event
, NULL
);
1313 /* Lower IRQL back */
1314 KeLowerIrql(OldIrql
);
1319 _ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex
)
1322 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1324 /* Raise to APC_LEVEL */
1325 KeRaiseIrql(APC_LEVEL
, &OldIrql
);
1327 /* Check if we can quickly acquire it */
1328 if (InterlockedCompareExchange(&FastMutex
->Count
, 0, 1) == 1)
1330 /* We have, set us as owners */
1331 FastMutex
->Owner
= KeGetCurrentThread();
1332 FastMutex
->OldIrql
= OldIrql
;
1337 /* Acquire attempt failed */
1338 KeLowerIrql(OldIrql
);
1346 _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex
)
1348 /* Enter the Critical Region */
1349 KeEnterCriticalRegion();
1351 /* Acquire the mutex unsafely */
1352 _ExAcquireFastMutexUnsafe(FastMutex
);
1357 _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex
)
1359 /* Release the mutex unsafely */
1360 _ExReleaseFastMutexUnsafe(FastMutex
);
1362 /* Leave the critical region */
1363 KeLeaveCriticalRegion();
1366 /* OTHER FUNCTIONS **********************************************************/
1370 ExTryToAcquireResourceExclusiveLite(
1371 IN PERESOURCE Resource
1375 ExpSetTimeZoneInformation(PTIME_ZONE_INFORMATION TimeZoneInformation
);
1379 ExAcquireTimeRefreshLock(BOOLEAN Wait
);
1383 ExReleaseTimeRefreshLock(VOID
);
1387 ExUpdateSystemTimeFromCmos(IN BOOLEAN UpdateInterruptTime
,
1388 IN ULONG MaxSepInSeconds
);
1392 ExpAllocateLocallyUniqueId(OUT LUID
*LocallyUniqueId
);
1396 ExTimerRundown(VOID
);
1401 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1407 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1410 #define InterlockedDecrementUL(Addend) \
1411 (ULONG)InterlockedDecrement((PLONG)(Addend))
1413 #define InterlockedIncrementUL(Addend) \
1414 (ULONG)InterlockedIncrement((PLONG)(Addend))
1416 #define InterlockedExchangeUL(Target, Value) \
1417 (ULONG)InterlockedExchange((PLONG)(Target), (LONG)(Value))
1419 #define InterlockedExchangeAddUL(Addend, Value) \
1420 (ULONG)InterlockedExchangeAdd((PLONG)(Addend), (LONG)(Value))
1422 #define InterlockedCompareExchangeUL(Destination, Exchange, Comperand) \
1423 (ULONG)InterlockedCompareExchange((PLONG)(Destination), (LONG)(Exchange), (LONG)(Comperand))
1425 #define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
1426 (ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))