Make definitions of ExpChangeRundown, ExpChangePushlock and ExpSetRundown 64 bit safe (have one portable definition and use it accordingly)
svn path=/trunk/; revision=50152
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Reset the count */
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Reset the count */
- ExpSetRundown(&RunRef->Count, 0);
+ ExpSetRundown(RunRef, 0);
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Mark the counter as active */
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Mark the counter as active */
- ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
+ ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
WaitBlock.Count = Count;
/* Now set the pointer */
WaitBlock.Count = Count;
/* Now set the pointer */
- NewValue = ExpChangeRundown(RunRef, PtrToUlong(WaitBlockPointer), Value);
+ NewValue = ExpChangeRundown(RunRef, (ULONG_PTR)WaitBlockPointer, Value);
if (NewValue == Value) break;
/* Loop again */
if (NewValue == Value) break;
/* Loop again */
// Detect old GCC
//
#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
// Detect old GCC
//
#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ < 40300) || \
- (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
+ (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ == 40303)
#define DEFINE_WAIT_BLOCK(x) \
struct _AlignHack \
#define DEFINE_WAIT_BLOCK(x) \
struct _AlignHack \
-#ifdef _WIN64
-#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
+#define ExpChangeRundown(x, y, z) (ULONG_PTR)InterlockedCompareExchangePointer(&x->Ptr, (PVOID)y, (PVOID)z)
#define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
#define ExpChangePushlock(x, y, z) InterlockedCompareExchangePointer((PVOID*)x, (PVOID)y, (PVOID)z)
-#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
-#else
-#define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
-#define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
-#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
-#endif
+#define ExpSetRundown(x, y) InterlockedExchangePointer(&x->Ptr, (PVOID)y)
/* INITIALIZATION FUNCTIONS *************************************************/
/* INITIALIZATION FUNCTIONS *************************************************/
{
/* Sanity check */
ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
{
/* Sanity check */
ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
/* Check if an object is being set */
if (!Object)
{
/* Check if an object is being set */
if (!Object)
{
ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
{
EX_FAST_REF OldValue, NewValue;
ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
{
EX_FAST_REF OldValue, NewValue;
/* Start reference loop */
for (;;)
{
/* Start reference loop */
for (;;)
{
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
}
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
}
/* We are done */
break;
}
/* We are done */
break;
}
/* Return the old value */
return OldValue;
}
/* Return the old value */
return OldValue;
}
IN PVOID Object)
{
EX_FAST_REF OldValue, NewValue;
IN PVOID Object)
{
EX_FAST_REF OldValue, NewValue;
/* Sanity checks */
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
/* Sanity checks */
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
/* Start update loop */
for (;;)
{
/* Get the current reference count */
OldValue = *FastRef;
/* Start update loop */
for (;;)
{
/* Get the current reference count */
OldValue = *FastRef;
/* Check if the current count is too high or if the pointer changed */
if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
/* Check if the current count is too high or if the pointer changed */
if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
/* Fail */
return FALSE;
}
/* Fail */
return FALSE;
}
/* Update the reference count */
NewValue.Value = OldValue.Value + MAX_FAST_REFS;
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* Update the reference count */
NewValue.Value = OldValue.Value + MAX_FAST_REFS;
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* We are done */
break;
}
/* We are done */
break;
}
IN PVOID Object)
{
EX_FAST_REF OldValue, NewValue;
IN PVOID Object)
{
EX_FAST_REF OldValue, NewValue;
/* Sanity checks */
ASSERT(Object != NULL);
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
/* Sanity checks */
ASSERT(Object != NULL);
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
/* Start reference loop */
for (;;)
{
/* Get the current reference count */
OldValue = *FastRef;
/* Start reference loop */
for (;;)
{
/* Get the current reference count */
OldValue = *FastRef;
/* Check if we're full if if the pointer changed */
if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
/* Check if we're full if if the pointer changed */
if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
/* Decrease the reference count */
NewValue.Value = OldValue.Value + 1;
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* Decrease the reference count */
NewValue.Value = OldValue.Value + 1;
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* We are done */
break;
}
/* We are done */
break;
}
/* Return success */
return TRUE;
}
/* Return success */
return TRUE;
}
IN PVOID Object)
{
EX_FAST_REF NewValue, OldValue;
IN PVOID Object)
{
EX_FAST_REF NewValue, OldValue;
/* Sanity check */
ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
/* Sanity check */
ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
/* Check if an object is being set */
if (!Object)
{
/* Check if an object is being set */
if (!Object)
{
/* Otherwise, we assume the object was referenced and is ready */
NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
}
/* Otherwise, we assume the object was referenced and is ready */
NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
}
/* Update the object */
OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
return OldValue;
/* Update the object */
OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
return OldValue;
IN PVOID OldObject)
{
EX_FAST_REF OldValue, NewValue;
IN PVOID OldObject)
{
EX_FAST_REF OldValue, NewValue;
/* Sanity check and start swap loop */
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
for (;;)
{
/* Get the current value */
OldValue = *FastRef;
/* Sanity check and start swap loop */
ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
for (;;)
{
/* Get the current value */
OldValue = *FastRef;
/* Make sure there's enough references to swap */
if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
/* Make sure there's enough references to swap */
if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
/* Check if we have an object to swap */
if (Object)
{
/* Check if we have an object to swap */
if (Object)
{
/* Write the object address itself (which is empty) */
NewValue.Value = (ULONG_PTR)Object;
}
/* Write the object address itself (which is empty) */
NewValue.Value = (ULONG_PTR)Object;
}
/* Do the actual compare exchange */
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* Do the actual compare exchange */
NewValue.Object = ExpChangePushlock(&FastRef->Object,
NewValue.Object,
OldValue.Object);
if (NewValue.Object != OldValue.Object) continue;
/* Return the old value */
return OldValue;
}
/* Return the old value */
return OldValue;
}
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Mark the counter as active */
ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
/* Mark the counter as active */
- ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
+ ExpSetRundown(RunRef, EX_RUNDOWN_ACTIVE);
}
/* PUSHLOCKS *****************************************************************/
}
/* PUSHLOCKS *****************************************************************/
_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
{
PKTHREAD Thread = KeGetCurrentThread();
_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
{
PKTHREAD Thread = KeGetCurrentThread();
/* Sanity check */
ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
(Thread->CombinedApcDisable != 0) ||
(Thread->Teb == NULL) ||
(Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
ASSERT(FastMutex->Owner != Thread);
/* Sanity check */
ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
(Thread->CombinedApcDisable != 0) ||
(Thread->Teb == NULL) ||
(Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
ASSERT(FastMutex->Owner != Thread);
/* Decrease the count */
if (InterlockedDecrement(&FastMutex->Count))
{
/* Someone is still holding it, use slow path */
KiAcquireFastMutex(FastMutex);
}
/* Decrease the count */
if (InterlockedDecrement(&FastMutex->Count))
{
/* Someone is still holding it, use slow path */
KiAcquireFastMutex(FastMutex);
}
/* Set the owner */
FastMutex->Owner = Thread;
}
/* Set the owner */
FastMutex->Owner = Thread;
}
(KeGetCurrentThread()->Teb == NULL) ||
(KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
ASSERT(FastMutex->Owner == KeGetCurrentThread());
(KeGetCurrentThread()->Teb == NULL) ||
(KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
ASSERT(FastMutex->Owner == KeGetCurrentThread());
/* Erase the owner */
FastMutex->Owner = NULL;
/* Erase the owner */
FastMutex->Owner = NULL;
/* Increase the count */
if (InterlockedIncrement(&FastMutex->Count) <= 0)
{
/* Increase the count */
if (InterlockedIncrement(&FastMutex->Count) <= 0)
{
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
/* Raise IRQL to APC */
KeRaiseIrql(APC_LEVEL, &OldIrql);
/* Raise IRQL to APC */
KeRaiseIrql(APC_LEVEL, &OldIrql);
/* Decrease the count */
if (InterlockedDecrement(&FastMutex->Count))
{
/* Someone is still holding it, use slow path */
KiAcquireFastMutex(FastMutex);
}
/* Decrease the count */
if (InterlockedDecrement(&FastMutex->Count))
{
/* Someone is still holding it, use slow path */
KiAcquireFastMutex(FastMutex);
}
/* Set the owner and IRQL */
FastMutex->Owner = KeGetCurrentThread();
FastMutex->OldIrql = OldIrql;
/* Set the owner and IRQL */
FastMutex->Owner = KeGetCurrentThread();
FastMutex->OldIrql = OldIrql;
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() == APC_LEVEL);
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() == APC_LEVEL);
/* Erase the owner */
FastMutex->Owner = NULL;
OldIrql = (KIRQL)FastMutex->OldIrql;
/* Erase the owner */
FastMutex->Owner = NULL;
OldIrql = (KIRQL)FastMutex->OldIrql;
/* Increase the count */
if (InterlockedIncrement(&FastMutex->Count) <= 0)
{
/* Someone was waiting for it, signal the waiter */
KeSetEventBoostPriority(&FastMutex->Event, NULL);
}
/* Increase the count */
if (InterlockedIncrement(&FastMutex->Count) <= 0)
{
/* Someone was waiting for it, signal the waiter */
KeSetEventBoostPriority(&FastMutex->Event, NULL);
}
/* Lower IRQL back */
KeLowerIrql(OldIrql);
}
/* Lower IRQL back */
KeLowerIrql(OldIrql);
}
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
{
KIRQL OldIrql;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
/* Raise to APC_LEVEL */
KeRaiseIrql(APC_LEVEL, &OldIrql);
/* Raise to APC_LEVEL */
KeRaiseIrql(APC_LEVEL, &OldIrql);
/* Check if we can quickly acquire it */
if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
{
/* Check if we can quickly acquire it */
if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
{