-#ifndef __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
-#define __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H
+#pragma once
/* GLOBAL VARIABLES *********************************************************/
extern LIST_ENTRY ExpFirmwareTableProviderListHead;
extern BOOLEAN ExpIsWinPEMode;
extern LIST_ENTRY ExpSystemResourcesList;
-ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
-ULONG ExpUnicodeCaseTableDataOffset;
-PVOID ExpNlsSectionPointer;
+extern ULONG ExpAnsiCodePageDataOffset, ExpOemCodePageDataOffset;
+extern ULONG ExpUnicodeCaseTableDataOffset;
+extern PVOID ExpNlsSectionPointer;
extern CHAR NtBuildLab[];
extern ULONG CmNtCSDVersion;
extern ULONG NtGlobalFlag;
extern ULONG ExpInitializationPhase;
extern ULONG ExpAltTimeZoneBias;
extern LIST_ENTRY ExSystemLookasideListHead;
+extern PCALLBACK_OBJECT PowerStateCallback;
typedef struct _EXHANDLE
{
//
// Detect old GCC
//
-#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300)
+#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
+ (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
#define DEFINE_WAIT_BLOCK(x) \
struct _AlignHack \
#endif
+#ifdef _WIN64
+#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
+#define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
+#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
+#else
+#define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+#define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
+#endif
+
/* INITIALIZATION FUNCTIONS *************************************************/
VOID
NTAPI
ExpInitializeWorkerThreads(VOID);
+VOID
+NTAPI
+ExSwapinWorkerThreads(IN BOOLEAN AllowSwap);
+
VOID
NTAPI
ExpInitLookasideLists(VOID);
IN PVOID Argument1,
IN PVOID Argument2)
{
- PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock;
+ PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
PEX_CALLBACK_FUNCTION Function;
/* Reference the block */
- CallbackRoutineBlock = ExReferenceCallBackBlock(Callback);
- if (CallbackRoutineBlock)
+ CallbackBlock = ExReferenceCallBackBlock(Callback);
+ if (CallbackBlock)
{
/* Get the function */
- Function = ExGetCallBackBlockRoutine(CallbackRoutineBlock);
+ Function = ExGetCallBackBlockRoutine(CallbackBlock);
/* Do the callback */
Function(Context, Argument1, Argument2);
/* Now dereference it */
- ExDereferenceCallBackBlock(Callback, CallbackRoutineBlock);
+ ExDereferenceCallBackBlock(Callback, CallbackBlock);
}
}
-/* RUNDOWN *******************************************************************/
+/* FAST REFS ******************************************************************/
-#ifdef _WIN64
-#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
-#define ExpChangePushlock(lock, new, old) InterlockedCompareExchangePointer((PVOID*)lock, (PVOID)new, (PVOID)old)
-#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
-#else
-#define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
-#define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
-#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
-#endif
+FORCEINLINE
+PVOID
+ExGetObjectFastReference(IN EX_FAST_REF FastRef)
+{
+ /* Return the unbiased pointer */
+ return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
+}
+
+FORCEINLINE
+ULONG
+ExGetCountFastReference(IN EX_FAST_REF FastRef)
+{
+ /* Return the reference count */
+ return FastRef.RefCnt;
+}
+
+FORCEINLINE
+VOID
+ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
+ IN OPTIONAL PVOID Object)
+{
+ /* Sanity check */
+ ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
+
+ /* Check if an object is being set */
+ if (!Object)
+ {
+ /* Clear the field */
+ FastRef->Object = NULL;
+ }
+ else
+ {
+ /* Otherwise, we assume the object was referenced and is ready */
+ FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+ }
+}
+
+FORCEINLINE
+EX_FAST_REF
+ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
+{
+ EX_FAST_REF OldValue, NewValue;
+
+ /* Start reference loop */
+ for (;;)
+ {
+ /* Get the current reference count */
+ OldValue = *FastRef;
+ if (OldValue.RefCnt)
+ {
+ /* Increase the reference count */
+ NewValue.Value = OldValue.Value - 1;
+ NewValue.Object = ExpChangePushlock(&FastRef->Object,
+ NewValue.Object,
+ OldValue.Object);
+ if (NewValue.Object != OldValue.Object) continue;
+ }
+
+ /* We are done */
+ break;
+ }
+
+ /* Return the old value */
+ return OldValue;
+}
+
+FORCEINLINE
+BOOLEAN
+ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
+ IN PVOID Object)
+{
+ EX_FAST_REF OldValue, NewValue;
+
+ /* Sanity checks */
+ ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+
+ /* Start update loop */
+ for (;;)
+ {
+ /* Get the current reference count */
+ OldValue = *FastRef;
+
+ /* Check if the current count is too high or if the pointer changed */
+ if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
+ ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
+ {
+ /* Fail */
+ return FALSE;
+ }
+
+ /* Update the reference count */
+ NewValue.Value = OldValue.Value + MAX_FAST_REFS;
+ NewValue.Object = ExpChangePushlock(&FastRef->Object,
+ NewValue.Object,
+ OldValue.Object);
+ if (NewValue.Object != OldValue.Object) continue;
+
+ /* We are done */
+ break;
+ }
+
+ /* Return success */
+ return TRUE;
+}
+
+FORCEINLINE
+BOOLEAN
+ExReleaseFastReference(IN PEX_FAST_REF FastRef,
+ IN PVOID Object)
+{
+ EX_FAST_REF OldValue, NewValue;
+
+ /* Sanity checks */
+ ASSERT(Object != NULL);
+ ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+
+ /* Start reference loop */
+ for (;;)
+ {
+ /* Get the current reference count */
+ OldValue = *FastRef;
+
+ /* Check if we're full if if the pointer changed */
+ if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
+
+ /* Decrease the reference count */
+ NewValue.Value = OldValue.Value + 1;
+ NewValue.Object = ExpChangePushlock(&FastRef->Object,
+ NewValue.Object,
+ OldValue.Object);
+ if (NewValue.Object != OldValue.Object) continue;
+
+ /* We are done */
+ break;
+ }
+
+ /* Return success */
+ return TRUE;
+}
+
+FORCEINLINE
+EX_FAST_REF
+ExSwapFastReference(IN PEX_FAST_REF FastRef,
+ IN PVOID Object)
+{
+ EX_FAST_REF NewValue, OldValue;
+
+ /* Sanity check */
+ ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
+
+ /* Check if an object is being set */
+ if (!Object)
+ {
+ /* Clear the field */
+ NewValue.Object = NULL;
+ }
+ else
+ {
+ /* Otherwise, we assume the object was referenced and is ready */
+ NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+ }
+
+ /* Update the object */
+ OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
+ return OldValue;
+}
+
+FORCEINLINE
+EX_FAST_REF
+ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
+ IN PVOID Object,
+ IN PVOID OldObject)
+{
+ EX_FAST_REF OldValue, NewValue;
+
+ /* Sanity check and start swap loop */
+ ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+ for (;;)
+ {
+ /* Get the current value */
+ OldValue = *FastRef;
+
+ /* Make sure there's enough references to swap */
+ if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
+
+ /* Check if we have an object to swap */
+ if (Object)
+ {
+ /* Set up the value with maximum fast references */
+ NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+ }
+ else
+ {
+ /* Write the object address itself (which is empty) */
+ NewValue.Value = (ULONG_PTR)Object;
+ }
+
+ /* Do the actual compare exchange */
+ NewValue.Object = ExpChangePushlock(&FastRef->Object,
+ NewValue.Object,
+ OldValue.Object);
+ if (NewValue.Object != OldValue.Object) continue;
+
+ /* All done */
+ break;
+ }
+
+ /* Return the old value */
+ return OldValue;
+}
+
+/* RUNDOWN *******************************************************************/
/*++
* @name ExfAcquireRundownProtection
);
/*++
- * @name ExInitializePushLock
+ * @name _ExInitializePushLock
* INTERNAL MACRO
*
- * The ExInitializePushLock macro initializes a PushLock.
+ * The _ExInitializePushLock macro initializes a PushLock.
*
* @params PushLock
* Pointer to the pushlock which is to be initialized.
*--*/
FORCEINLINE
VOID
-ExInitializePushLock(IN PULONG_PTR PushLock)
+_ExInitializePushLock(IN PULONG_PTR PushLock)
{
/* Set the value to 0 */
*PushLock = 0;
}
+#define ExInitializePushLock _ExInitializePushLock
/*++
* @name ExAcquirePushLockExclusive
}
}
+/* FAST MUTEX INLINES *********************************************************/
+
+FORCEINLINE
+VOID
+_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
+{
+ PKTHREAD Thread = KeGetCurrentThread();
+
+ /* Sanity check */
+ ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+ (Thread->CombinedApcDisable != 0) ||
+ (Thread->Teb == NULL) ||
+ (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+ ASSERT(FastMutex->Owner != Thread);
+
+ /* Decrease the count */
+ if (InterlockedDecrement(&FastMutex->Count))
+ {
+ /* Someone is still holding it, use slow path */
+ KiAcquireFastMutex(FastMutex);
+ }
+
+ /* Set the owner */
+ FastMutex->Owner = Thread;
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
+{
+ ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+ (KeGetCurrentThread()->CombinedApcDisable != 0) ||
+ (KeGetCurrentThread()->Teb == NULL) ||
+ (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+ ASSERT(FastMutex->Owner == KeGetCurrentThread());
+
+ /* Erase the owner */
+ FastMutex->Owner = NULL;
+
+ /* Increase the count */
+ if (InterlockedIncrement(&FastMutex->Count) <= 0)
+ {
+ /* Someone was waiting for it, signal the waiter */
+ KeSetEventBoostPriority(&FastMutex->Event, NULL);
+ }
+}
+
+FORCEINLINE
+VOID
+_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
+{
+ KIRQL OldIrql;
+ ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+
+ /* Raise IRQL to APC */
+ KeRaiseIrql(APC_LEVEL, &OldIrql);
+
+ /* Decrease the count */
+ if (InterlockedDecrement(&FastMutex->Count))
+ {
+ /* Someone is still holding it, use slow path */
+ KiAcquireFastMutex(FastMutex);
+ }
+
+ /* Set the owner and IRQL */
+ FastMutex->Owner = KeGetCurrentThread();
+ FastMutex->OldIrql = OldIrql;
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
+{
+ KIRQL OldIrql;
+ ASSERT(KeGetCurrentIrql() == APC_LEVEL);
+
+ /* Erase the owner */
+ FastMutex->Owner = NULL;
+ OldIrql = (KIRQL)FastMutex->OldIrql;
+
+ /* Increase the count */
+ if (InterlockedIncrement(&FastMutex->Count) <= 0)
+ {
+ /* Someone was waiting for it, signal the waiter */
+ KeSetEventBoostPriority(&FastMutex->Event, NULL);
+ }
+
+ /* Lower IRQL back */
+ KeLowerIrql(OldIrql);
+}
+
+FORCEINLINE
+BOOLEAN
+_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
+{
+ KIRQL OldIrql;
+ ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+
+ /* Raise to APC_LEVEL */
+ KeRaiseIrql(APC_LEVEL, &OldIrql);
+
+ /* Check if we can quickly acquire it */
+ if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
+ {
+ /* We have, set us as owners */
+ FastMutex->Owner = KeGetCurrentThread();
+ FastMutex->OldIrql = OldIrql;
+ return TRUE;
+ }
+ else
+ {
+ /* Acquire attempt failed */
+ KeLowerIrql(OldIrql);
+ YieldProcessor();
+ return FALSE;
+ }
+}
+
+FORCEINLINE
+VOID
+_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
+{
+ /* Enter the Critical Region */
+ KeEnterCriticalRegion();
+
+ /* Acquire the mutex unsafely */
+ _ExAcquireFastMutexUnsafe(FastMutex);
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
+{
+ /* Release the mutex unsafely */
+ _ExReleaseFastMutexUnsafe(FastMutex);
+
+ /* Leave the critical region */
+ KeLeaveCriticalRegion();
+}
+
/* OTHER FUNCTIONS **********************************************************/
BOOLEAN
#define ExfInterlockedCompareExchange64UL(Destination, Exchange, Comperand) \
(ULONGLONG)ExfInterlockedCompareExchange64((PLONGLONG)(Destination), (PLONGLONG)(Exchange), (PLONGLONG)(Comperand))
-
-#endif /* __NTOSKRNL_INCLUDE_INTERNAL_EXECUTIVE_H */
projects / reactos.git / blobdiff