#include <ntstatus.h>
#include <kernelspecs.h>
#include <ntiologc.h>
+#include <suppress.h>
#ifndef GUID_DEFINED
#include <guiddef.h>
extern "C" {
#endif
+
#if !defined(_NTHALDLL_) && !defined(_BLDR_)
#define NTHALAPI DECLSPEC_IMPORT
#else
#define BitTestAndComplement _bittestandcomplement
#define BitTestAndSet _bittestandset
#define BitTestAndReset _bittestandreset
-#define InterlockedBitTestAndSet _interlockedbittestandset
-#define InterlockedBitTestAndReset _interlockedbittestandreset
-
-#ifdef _M_AMD64
+#ifdef _WIN64
#define BitScanForward64 _BitScanForward64
#define BitScanReverse64 _BitScanReverse64
#define BitTest64 _bittest64
#define BitTestAndComplement64 _bittestandcomplement64
#define BitTestAndSet64 _bittestandset64
#define BitTestAndReset64 _bittestandreset64
+#endif /* _WIN64 */
+
+#if defined(_M_ARM) || defined(_M_IA64)
+#define __ACQ_(x) x##_acq
+#define __REL_(x) x##_rel
+#define __NF_(x) x##_nf
+#else
+#define __ACQ_(x) x
+#define __REL_(x) x
+#define __NF_(x) x
+#endif
+
+#define InterlockedBitTestAndSet _interlockedbittestandset
+#define InterlockedBitTestAndSetAcquire __ACQ_(_interlockedbittestandset)
+#define InterlockedBitTestAndSetRelease __REL_(_interlockedbittestandset)
+#define InterlockedBitTestAndSetNoFence __NF_(_interlockedbittestandset)
+
+#define InterlockedBitTestAndReset _interlockedbittestandreset
+#define InterlockedBitTestAndResetAcquire __ACQ_(_interlockedbittestandreset)
+#define InterlockedBitTestAndResetRelease __REL_(_interlockedbittestandreset)
+#define InterlockedBitTestAndResetNoFence __NF_(_interlockedbittestandreset)
+
+#ifdef _WIN64
#define InterlockedBitTestAndSet64 _interlockedbittestandset64
+#define InterlockedBitTestAndSet64Acquire __ACQ_(_interlockedbittestandset64)
+#define InterlockedBitTestAndSet64Release __REL_(_interlockedbittestandset64)
+#define InterlockedBitTestAndSet64NoFence __NF_(_interlockedbittestandset64)
+
#define InterlockedBitTestAndReset64 _interlockedbittestandreset64
-#endif
+#define InterlockedBitTestAndReset64Acquire __ACQ_(_interlockedbittestandreset64)
+#define InterlockedBitTestAndReset64Release __REL_(_interlockedbittestandreset64)
+#define InterlockedBitTestAndReset64NoFence __NF_(_interlockedbittestandreset64)
+#endif /* _WIN64 */
-#if !defined(__INTERLOCKED_DECLARED)
-#define __INTERLOCKED_DECLARED
+#define InterlockedAdd _InterlockedAdd
+#define InterlockedAddAcquire __ACQ_(_InterlockedAdd)
+#define InterlockedAddRelease __REL_(_InterlockedAdd)
+#define InterlockedAddNoFence __NF_(_InterlockedAdd)
-#if defined (_X86_)
-#if defined(NO_INTERLOCKED_INTRINSICS)
-NTKERNELAPI
-LONG
-FASTCALL
-InterlockedIncrement(
- _Inout_ _Interlocked_operand_ LONG volatile *Addend);
+#define InterlockedAdd64 _InterlockedAdd64
+#define InterlockedAddAcquire64 __ACQ_(_InterlockedAdd64)
+#define InterlockedAddRelease64 __REL_(_InterlockedAdd64)
+#define InterlockedAddNoFence64 __NF_(_InterlockedAdd64)
-NTKERNELAPI
-LONG
-FASTCALL
-InterlockedDecrement(
- _Inout_ _Interlocked_operand_ LONG volatile *Addend);
+#define InterlockedAnd _InterlockedAnd
+#define InterlockedAndAcquire __ACQ_(_InterlockedAnd)
+#define InterlockedAndRelease __REL_(_InterlockedAnd)
+#define InterlockedAndNoFence __NF_(_InterlockedAnd)
+
+#define InterlockedAnd8 _InterlockedAnd8
+#ifdef _M_ARM
+#define InterlockedAndAcquire8 _InterlockedAnd8_acq
+#define InterlockedAndRelease8 _InterlockedAnd8_rel
+#define InterlockedAndNoFence8 _InterlockedAnd8_nf
+#elif defined(_M_IA64)
+#define InterlockedAnd8Acquire _InterlockedAnd8_acq
+#define InterlockedAnd8Release _InterlockedAnd8_rel
+#endif // _M_ARM
+
+#define InterlockedAnd16 _InterlockedAnd16
+#ifdef _M_ARM
+#define InterlockedAndAcquire16 _InterlockedAnd16_acq
+#define InterlockedAndRelease16 _InterlockedAnd16_rel
+#define InterlockedAndNoFence16 _InterlockedAnd16_nf
+#elif defined(_M_IA64)
+#define InterlockedAnd16Acquire _InterlockedAnd16_acq
+#define InterlockedAnd16Release _InterlockedAnd16_rel
+#endif // _M_ARM
-NTKERNELAPI
-LONG
-FASTCALL
-InterlockedCompareExchange(
- _Inout_ _Interlocked_operand_ LONG volatile *Destination,
- _In_ LONG Exchange,
- _In_ LONG Comparand);
+#define InterlockedAnd64 _InterlockedAnd64
+#ifdef _M_ARM
+#define InterlockedAndAcquire64 __ACQ_(_InterlockedAnd64)
+#define InterlockedAndRelease64 __REL_(_InterlockedAnd64)
+#define InterlockedAndNoFence64 __NF_(_InterlockedAnd64)
+#else // _M_ARM
+#define InterlockedAnd64Acquire __ACQ_(_InterlockedAnd64)
+#define InterlockedAnd64Release __REL_(_InterlockedAnd64)
+#define InterlockedAnd64NoFence __NF_(_InterlockedAnd64)
+#endif // _M_ARM
-NTKERNELAPI
-LONG
-FASTCALL
-InterlockedExchange(
- _Inout_ _Interlocked_operand_ LONG volatile *Destination,
- _In_ LONG Value);
+#ifdef _WIN64
+#define InterlockedAndAffinity InterlockedAnd64
+#else
+#define InterlockedAndAffinity InterlockedAnd
+#endif // _WIN64
-NTKERNELAPI
-LONG
-FASTCALL
-InterlockedExchangeAdd(
- _Inout_ _Interlocked_operand_ LONG volatile *Addend,
- _In_ LONG Value);
+#define InterlockedCompareExchange _InterlockedCompareExchange
+#define InterlockedCompareExchangeAcquire __ACQ_(_InterlockedCompareExchange)
+#define InterlockedCompareExchangeRelease __REL_(_InterlockedCompareExchange)
+#define InterlockedCompareExchangeNoFence __NF_(_InterlockedCompareExchange)
-#else /* !defined(NO_INTERLOCKED_INTRINSICS) */
+#define InterlockedCompareExchange16 _InterlockedCompareExchange16
+#define InterlockedCompareExchangeAcquire16 __ACQ_(_InterlockedCompareExchange16)
+#define InterlockedCompareExchangeRelease16 __REL_(_InterlockedCompareExchange16)
+#define InterlockedCompareExchangeNoFence16 __NF_(_InterlockedCompareExchange16)
+
+#define InterlockedCompareExchange64 _InterlockedCompareExchange64
+#define InterlockedCompareExchangeAcquire64 __ACQ_(_InterlockedCompareExchange64)
+#define InterlockedCompareExchangeRelease64 __REL_(_InterlockedCompareExchange64)
+#define InterlockedCompareExchangeNoFence64 __NF_(_InterlockedCompareExchange64)
+
+#ifdef _WIN64
+#define InterlockedCompareExchange128 _InterlockedCompareExchange128
+#endif // _WIN64
+
+#ifdef _M_IA64
+#define InterlockedCompare64Exchange128 _InterlockedCompare64Exchange128
+#define InterlockedCompare64ExchangeAcquire128 _InterlockedCompare64Exchange128_acq
+#define InterlockedCompare64ExchangeRelease128 _InterlockedCompare64Exchange128_rel
+#endif // _M_IA64
+
+#define InterlockedCompareExchangePointer _InterlockedCompareExchangePointer
+#define InterlockedCompareExchangePointerAcquire __ACQ_(_InterlockedCompareExchangePointer)
+#define InterlockedCompareExchangePointerRelease __REL_(_InterlockedCompareExchangePointer)
+#define InterlockedCompareExchangePointerNoFence __NF_(_InterlockedCompareExchangePointer)
-#define InterlockedExchange _InterlockedExchange
-#define InterlockedIncrement _InterlockedIncrement
#define InterlockedDecrement _InterlockedDecrement
-#define InterlockedExchangeAdd _InterlockedExchangeAdd
-#define InterlockedCompareExchange _InterlockedCompareExchange
-#define InterlockedOr _InterlockedOr
-#define InterlockedAnd _InterlockedAnd
-#define InterlockedXor _InterlockedXor
+#define InterlockedDecrementAcquire __ACQ_(_InterlockedDecrement)
+#define InterlockedDecrementRelease __REL_(_InterlockedDecrement)
+#define InterlockedDecrementNoFence __NF_(_InterlockedDecrement)
-#endif /* !defined(NO_INTERLOCKED_INTRINSICS) */
+#define InterlockedDecrement16 _InterlockedDecrement16
+#define InterlockedDecrementAcquire16 __ACQ_(_InterlockedDecrement16)
+#define InterlockedDecrementRelease16 __REL_(_InterlockedDecrement16)
+#define InterlockedDecrementNoFence16 __NF_(_InterlockedDecrement16)
-#endif /* defined (_X86_) */
+#define InterlockedDecrement64 _InterlockedDecrement64
+#define InterlockedDecrementAcquire64 __ACQ_(_InterlockedDecrement64)
+#define InterlockedDecrementRelease64 __REL_(_InterlockedDecrement64)
+#define InterlockedDecrementNoFence64 __NF_(_InterlockedDecrement64)
-#if !defined (_WIN64)
-/*
- * PVOID
- * InterlockedExchangePointer(
- * IN OUT PVOID volatile *Target,
- * IN PVOID Value)
- */
-#define InterlockedExchangePointer(Target, Value) \
- ((PVOID) InterlockedExchange((PLONG) Target, (LONG) Value))
+#ifdef _WIN64
+#define InterlockedDecrementSizeT(a) InterlockedDecrement64((LONG64 *)a)
+#define InterlockedDecrementSizeTNoFence(a) InterlockedDecrementNoFence64((LONG64 *)a)
+#else
+#define InterlockedDecrementSizeT(a) InterlockedDecrement((LONG *)a)
+#define InterlockedDecrementSizeTNoFence(a) InterlockedDecrementNoFence((LONG *)a)
+#endif // _WIN64
-/*
- * PVOID
- * InterlockedCompareExchangePointer(
- * IN OUT PVOID *Destination,
- * IN PVOID Exchange,
- * IN PVOID Comparand)
- */
-#define InterlockedCompareExchangePointer(Destination, Exchange, Comparand) \
- ((PVOID) InterlockedCompareExchange((PLONG) Destination, (LONG) Exchange, (LONG) Comparand))
+#define InterlockedExchange _InterlockedExchange
+#define InterlockedExchangeAcquire __ACQ_(_InterlockedExchange)
+/* No release here */
+#define InterlockedExchangeNoFence __NF_(_InterlockedExchange)
+
+#if (_MSC_VER >= 1600)
+#define InterlockedExchange8 _InterlockedExchange8
+#endif // (_MSC_VER >= 1600)
+
+#define InterlockedExchange16 _InterlockedExchange16
+/* No release here */
+#define InterlockedExchangeAcquire16 __ACQ_(_InterlockedExchange16)
+#define InterlockedExchangeNoFence16 __NF_(_InterlockedExchange16)
+
+#define InterlockedExchange64 _InterlockedExchange64
+#define InterlockedExchangeAcquire64 __ACQ_(_InterlockedExchange64)
+/* No release here */
+#define InterlockedExchangeNoFence64 __NF_(_InterlockedExchange64)
+
+#define InterlockedExchangePointer _InterlockedExchangePointer
+#define InterlockedExchangePointerAcquire __ACQ_(_InterlockedExchangePointer)
+/* No release here */
+#define InterlockedExchangePointerNoFence __NF_(_InterlockedExchangePointer)
+
+#define InterlockedExchangeAdd _InterlockedExchangeAdd
+#define InterlockedExchangeAddAcquire __ACQ_(_InterlockedExchangeAdd)
+#define InterlockedExchangeAddRelease __REL_(_InterlockedExchangeAdd)
+#define InterlockedExchangeAddNoFence __NF_(_InterlockedExchangeAdd)
+
+#define InterlockedExchangeAdd64 _InterlockedExchangeAdd64
+#define InterlockedExchangeAddAcquire64 __ACQ_(_InterlockedExchangeAdd64)
+#define InterlockedExchangeAddRelease64 __REL_(_InterlockedExchangeAdd64)
+#define InterlockedExchangeAddNoFence64 __NF_(_InterlockedExchangeAdd64)
+#ifdef _WIN64
+#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((LONG64 *)a, b)
+#define InterlockedExchangeAddSizeTAcquire(a, b) InterlockedExchangeAddAcquire64((LONG64 *)a, b)
+#define InterlockedExchangeAddSizeTNoFence(a, b) InterlockedExchangeAddNoFence64((LONG64 *)a, b)
+#else
#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd((LONG *)a, b)
-#define InterlockedIncrementSizeT(a) InterlockedIncrement((LONG *)a)
-#define InterlockedDecrementSizeT(a) InterlockedDecrement((LONG *)a)
+#define InterlockedExchangeAddSizeTAcquire(a, b) InterlockedExchangeAddAcquire((LONG *)a, b)
+#define InterlockedExchangeAddSizeTNoFence(a, b) InterlockedExchangeAddNoFence((LONG *)a, b)
+#endif // _WIN64
+
+#define InterlockedIncrement _InterlockedIncrement
+#define InterlockedIncrementAcquire __ACQ_(_InterlockedIncrement)
+#define InterlockedIncrementRelease __REL_(_InterlockedIncrement)
+#define InterlockedIncrementNoFence __NF_(_InterlockedIncrement)
-#endif // !defined (_WIN64)
+#define InterlockedIncrement16 _InterlockedIncrement16
+#define InterlockedIncrementAcquire16 __ACQ_(_InterlockedIncrement16)
+#define InterlockedIncrementRelease16 __REL_(_InterlockedIncrement16)
+#define InterlockedIncrementNoFence16 __NF_(_InterlockedIncrement16)
-#if defined (_M_AMD64)
+#define InterlockedIncrement64 _InterlockedIncrement64
+#define InterlockedIncrementAcquire64 __ACQ_(_InterlockedIncrement64)
+#define InterlockedIncrementRelease64 __REL_(_InterlockedIncrement64)
+#define InterlockedIncrementNoFence64 __NF_(_InterlockedIncrement64)
+
+#ifdef _WIN64
+#define InterlockedIncrementSizeT(a) InterlockedIncrement64((LONG64 *)a)
+#define InterlockedIncrementSizeTNoFence(a) InterlockedIncrementNoFence64((LONG64 *)a)
+#else
+#define InterlockedIncrementSizeT(a) InterlockedIncrement((LONG *)a)
+#define InterlockedIncrementSizeTNoFence(a) InterlockedIncrementNoFence((LONG *)a)
+#endif // _WIN64
-#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((LONGLONG *)a, (LONGLONG)b)
-#define InterlockedIncrementSizeT(a) InterlockedIncrement64((LONGLONG *)a)
-#define InterlockedDecrementSizeT(a) InterlockedDecrement64((LONGLONG *)a)
-#define InterlockedAnd _InterlockedAnd
#define InterlockedOr _InterlockedOr
-#define InterlockedXor _InterlockedXor
-#define InterlockedIncrement _InterlockedIncrement
-#define InterlockedDecrement _InterlockedDecrement
-#define InterlockedAdd _InterlockedAdd
-#define InterlockedExchange _InterlockedExchange
-#define InterlockedExchangeAdd _InterlockedExchangeAdd
-#define InterlockedCompareExchange _InterlockedCompareExchange
-#define InterlockedAnd64 _InterlockedAnd64
+#define InterlockedOrAcquire __ACQ_(_InterlockedOr)
+#define InterlockedOrRelease __REL_(_InterlockedOr)
+#define InterlockedOrNoFence __NF_(_InterlockedOr)
+
+#define InterlockedOr8 _InterlockedOr8
+#ifdef _M_ARM
+#define InterlockedOrAcquire8 _InterlockedOr8_acq
+#define InterlockedOrRelease8 _InterlockedOr8_rel
+#define InterlockedOrNoFence8 _InterlockedOr8_nf
+#elif defined(_M_IA64)
+#define InterlockedOr8Acquire _InterlockedOr8_acq
+#define InterlockedOr8Release _InterlockedOr8_rel
+#endif // _M_ARM
+
+#define InterlockedOr16 _InterlockedOr16
+#ifdef _M_ARM
+#define InterlockedOrAcquire16 _InterlockedOr16_acq
+#define InterlockedOrRelease16 _InterlockedOr16_rel
+#define InterlockedOrNoFence16 _InterlockedOr16_nf
+#elif defined(_M_IA64)
+#define InterlockedOr16Acquire _InterlockedOr16_acq
+#define InterlockedOr16Release _InterlockedOr16_rel
+#endif // _M_ARM
+
#define InterlockedOr64 _InterlockedOr64
+#ifdef _M_ARM
+#define InterlockedOrAcquire64 _InterlockedOr64_acq
+#define InterlockedOrRelease64 _InterlockedOr64_rel
+#define InterlockedOrNoFence64 _InterlockedOr64_nf
+#elif defined(_M_IA64) || defined(_M_AMD64)
+#define InterlockedOr64Acquire __ACQ_(_InterlockedOr64)
+#define InterlockedOr64Release __REL_(_InterlockedOr64)
+#define InterlockedOr64NoFence __NF_(_InterlockedOr64)
+#endif // _M_ARM
+
+#ifdef _WIN64
+#define InterlockedOrAffinity InterlockedOr64
+#else
+#define InterlockedOrAffinity InterlockedOr
+#endif // _WIN64
+
+#define InterlockedXor _InterlockedXor
+#define InterlockedXorAcquire __ACQ_(_InterlockedXor)
+#define InterlockedXorRelease __REL_(_InterlockedXor)
+#define InterlockedXorNoFence __NF_(_InterlockedXor)
+
+#define InterlockedXor8 _InterlockedXor8
+#ifdef _M_ARM
+#define InterlockedXorAcquire8 _InterlockedXor8_acq
+#define InterlockedXorRelease8 _InterlockedXor8_rel
+#define InterlockedXorNoFence8 _InterlockedXor8_nf
+#elif defined(_M_IA64)
+#define InterlockedXor8Acquire _InterlockedXor8_acq
+#define InterlockedXor8Release _InterlockedXor8_rel
+#endif /* _M_ARM */
+
+#define InterlockedXor16 _InterlockedXor16
+#ifdef _M_ARM
+#define InterlockedXorAcquire16 _InterlockedXor16_acq
+#define InterlockedXorRelease16 _InterlockedXor16_rel
+#define InterlockedXorNoFence16 _InterlockedXor16_nf
+#elif defined(_M_IA64)
+#define InterlockedXor16Acquire _InterlockedXor16_acq
+#define InterlockedXor16Release _InterlockedXor16_rel
+#endif /* _M_ARM */
+
#define InterlockedXor64 _InterlockedXor64
-#define InterlockedIncrement64 _InterlockedIncrement64
-#define InterlockedDecrement64 _InterlockedDecrement64
-#define InterlockedAdd64 _InterlockedAdd64
-#define InterlockedExchange64 _InterlockedExchange64
-#define InterlockedExchangeAdd64 _InterlockedExchangeAdd64
-#define InterlockedCompareExchange64 _InterlockedCompareExchange64
-#define InterlockedCompareExchangePointer _InterlockedCompareExchangePointer
-#define InterlockedExchangePointer _InterlockedExchangePointer
-#define InterlockedBitTestAndSet64 _interlockedbittestandset64
-#define InterlockedBitTestAndReset64 _interlockedbittestandreset64
+#ifdef _M_ARM
+#define InterlockedXorAcquire64 _InterlockedXor64_acq
+#define InterlockedXorRelease64 _InterlockedXor64_rel
+#define InterlockedXorNoFence64 _InterlockedXor64_nf
+#elif defined(_M_IA64) || defined(_M_AMD64)
+#define InterlockedXor64Acquire __ACQ_(_InterlockedXor64)
+#define InterlockedXor64Release __REL_(_InterlockedXor64)
+#define InterlockedXor64NoFence __NF_(_InterlockedXor64)
+#endif /* _M_ARM */
-#endif // _M_AMD64
+#ifdef _M_IX86
-#if defined(_M_AMD64) && !defined(RC_INVOKED) && !defined(MIDL_PASS)
-//#if !defined(_X86AMD64_) // FIXME: what's _X86AMD64_ used for?
FORCEINLINE
LONG64
-InterlockedAdd64(
- _Inout_ _Interlocked_operand_ LONG64 volatile *Addend,
+_InterlockedExchange64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
_In_ LONG64 Value)
{
- return InterlockedExchangeAdd64(Addend, Value) + Value;
+ LONG64 Old, Prev;
+ for (Old = *Target; ; Old = Prev)
+ {
+ Prev = _InterlockedCompareExchange64(Target, Value, Old);
+ if (Prev == Old)
+ return Prev;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedAdd64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value)
+{
+ LONG64 Old, Prev, New;
+ for (Old = *Target; ; Old = Prev)
+ {
+ New = Old + Value;
+ Prev = _InterlockedCompareExchange64(Target, New, Old);
+ if (Prev == Old)
+ return New;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedExchangeAdd64 (
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value
+ )
+{
+ LONG64 Old, Prev, New;
+ for (Old = *Target; ; Old = Prev)
+ {
+ New = Old + Value;
+ Prev = _InterlockedCompareExchange64(Target, New, Old);
+ if (Prev == Old)
+ return Prev;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedAnd64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value)
+{
+ LONG64 Old, Prev, New;
+ for (Old = *Target; ; Old = Prev)
+ {
+ New = Old & Value;
+ Prev = _InterlockedCompareExchange64(Target, New, Old);
+ if (Prev == Old)
+ return New;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedOr64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value)
+{
+ LONG64 Old, Prev, New;
+ for (Old = *Target; ; Old = Prev)
+ {
+ New = Old | Value;
+ Prev = _InterlockedCompareExchange64(Target, New, Old);
+ if (Prev == Old)
+ return New;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedXor64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value)
+{
+ LONG64 Old, Prev, New;
+ for (Old = *Target; ; Old = Prev)
+ {
+ New = Old ^ Value;
+ Prev = _InterlockedCompareExchange64(Target, New, Old);
+ if (Prev == Old)
+ return New;
+ }
+}
+
+FORCEINLINE
+LONG64
+_InterlockedIncrement64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target)
+{
+ return _InterlockedAdd64(Target, 1);
+}
+
+FORCEINLINE
+LONG64
+_InterlockedDecrement64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target)
+{
+ return _InterlockedAdd64(Target, -1);
+}
+
+#undef _InterlockedExchangePointer
+#define _InterlockedExchangePointer _InlineInterlockedExchangePointer
+FORCEINLINE
+_Ret_writes_(_Inexpressible_(Unknown))
+PVOID
+_InterlockedExchangePointer(
+ _Inout_ _At_(*Destination, _Pre_writable_byte_size_(_Inexpressible_(Unknown))
+ _Post_writable_byte_size_(_Inexpressible_(Unknown)))
+ _Interlocked_operand_ volatile PVOID *Destination,
+ _In_opt_ PVOID Value)
+{
+ return (PVOID)InterlockedExchange((volatile long *)Destination, (long)Value);
+}
+
+#undef _InterlockedCompareExchangePointer
+#define _InterlockedCompareExchangePointer _InlineInterlockedCompareExchangePointer
+FORCEINLINE
+_Ret_writes_(_Inexpressible_(Unknown))
+PVOID
+_InterlockedCompareExchangePointer(
+ _Inout_ _At_(*Destination, _Pre_writable_byte_size_(_Inexpressible_(Unknown))
+ _Post_writable_byte_size_(_Inexpressible_(Unknown)))
+ _Interlocked_operand_ volatile PVOID *Destination,
+ _In_opt_ PVOID ExChange,
+ _In_opt_ PVOID Comperand)
+{
+ return (PVOID)InterlockedCompareExchange((volatile long *)Destination,
+ (long)ExChange,
+ (long)Comperand);
+}
+
+#endif /* _M_IX86 */
+
+#ifdef _M_AMD64
+
+FORCEINLINE
+LONG64
+_InterlockedAdd64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Value)
+
+{
+ return _InterlockedExchangeAdd64(Target, Value) + Value;
+}
+
+#endif /* _M_AMD64 */
+
+#ifdef _M_IA64
+
+#undef _InterlockedBitTestAndSet
+#define _InterlockedBitTestAndSet InterlockedBitTestAndSet_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndSet(
+ _Inout_ _Interlocked_operand_ volatile LONG *Target,
+ _In_ LONG Bit)
+{
+ ULONG Mask = 1 << (Bit & 31);
+ return (BOOLEAN)((InterlockedOr(&Target[Bit / 32], Mask) & Mask) != 0);
+}
+
+#undef _InterlockedBitTestAndReset
+#define _InterlockedBitTestAndReset InterlockedBitTestAndReset_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndReset(
+ _Inout_ _Interlocked_operand_ volatile LONG *Target,
+ _In_ LONG Bit)
+{
+ ULONG Mask = 1 << (Bit & 31);
+ return (BOOLEAN)((InterlockedAnd(&Target[Bit / 32], ~Mask) & Mask) != 0);
+}
+
+#undef _InterlockedBitTestAndSet64
+#define _InterlockedBitTestAndSet64 InterlockedBitTestAndSet64_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndSet64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Bit)
+{
+ ULONG64 Mask = 1LL << (Bit & 63);
+ return (BOOLEAN)((InterlockedOr64(&Target[Bit / 64], Mask) & Mask) != 0);
+}
+
+#undef _InterlockedBitTestAndReset64
+#define _InterlockedBitTestAndReset64 InterlockedBitTestAndReset64_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndReset64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Bit)
+{
+ ULONG64 Mask = 1LL << (Bit & 63);
+ return (BOOLEAN)((InterlockedAnd64(&Target[Bit / 64], ~Mask) & Mask) != 0);
+}
+
+#undef _InterlockedBitTestAndComplement
+#define _InterlockedBitTestAndComplement InterlockedBitTestAndComplement_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndComplement(
+ _Inout_ _Interlocked_operand_ volatile LONG *Target,
+ _In_ LONG Bit)
+{
+ ULONG Mask = 1 << (Bit & 31);
+ return (BOOLEAN)((InterlockedXor(&Target[Bit / 32], Mask) & Mask) != 0);
}
-//#endif
-#endif
-#endif /* !__INTERLOCKED_DECLARED */
+#undef _InterlockedBitTestAndComplement64
+#define _InterlockedBitTestAndComplement64 InterlockedBitTestAndComplement64_Inline
+FORCEINLINE
+BOOLEAN
+_InterlockedBitTestAndComplement64(
+ _Inout_ _Interlocked_operand_ volatile LONG64 *Target,
+ _In_ LONG64 Bit)
+{
+ ULONG64 Mask = 1LL << (Bit & 63);
+ return (BOOLEAN)((InterlockedXor64(&Target[Bit / 64], Mask) & Mask) != 0);
+}
+#endif /* M_IA64 */
/******************************************************************************
* Runtime Library Types *
} SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX, *PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX;;
/* Processor features */
-#define PF_FLOATING_POINT_PRECISION_ERRATA 0
-#define PF_FLOATING_POINT_EMULATED 1
-#define PF_COMPARE_EXCHANGE_DOUBLE 2
-#define PF_MMX_INSTRUCTIONS_AVAILABLE 3
-#define PF_PPC_MOVEMEM_64BIT_OK 4
-#define PF_ALPHA_BYTE_INSTRUCTIONS 5
-#define PF_XMMI_INSTRUCTIONS_AVAILABLE 6
-#define PF_3DNOW_INSTRUCTIONS_AVAILABLE 7
-#define PF_RDTSC_INSTRUCTION_AVAILABLE 8
-#define PF_PAE_ENABLED 9
-#define PF_XMMI64_INSTRUCTIONS_AVAILABLE 10
-#define PF_SSE_DAZ_MODE_AVAILABLE 11
-#define PF_NX_ENABLED 12
-#define PF_SSE3_INSTRUCTIONS_AVAILABLE 13
-#define PF_COMPARE_EXCHANGE128 14
-#define PF_COMPARE64_EXCHANGE128 15
-#define PF_CHANNELS_ENABLED 16
-#define PF_XSAVE_ENABLED 17
+#define PF_FLOATING_POINT_PRECISION_ERRATA 0
+#define PF_FLOATING_POINT_EMULATED 1
+#define PF_COMPARE_EXCHANGE_DOUBLE 2
+#define PF_MMX_INSTRUCTIONS_AVAILABLE 3
+#define PF_PPC_MOVEMEM_64BIT_OK 4
+#define PF_ALPHA_BYTE_INSTRUCTIONS 5
+#define PF_XMMI_INSTRUCTIONS_AVAILABLE 6
+#define PF_3DNOW_INSTRUCTIONS_AVAILABLE 7
+#define PF_RDTSC_INSTRUCTION_AVAILABLE 8
+#define PF_PAE_ENABLED 9
+#define PF_XMMI64_INSTRUCTIONS_AVAILABLE 10
+#define PF_SSE_DAZ_MODE_AVAILABLE 11
+#define PF_NX_ENABLED 12
+#define PF_SSE3_INSTRUCTIONS_AVAILABLE 13
+#define PF_COMPARE_EXCHANGE128 14
+#define PF_COMPARE64_EXCHANGE128 15
+#define PF_CHANNELS_ENABLED 16
+#define PF_XSAVE_ENABLED 17
+#define PF_ARM_VFP_32_REGISTERS_AVAILABLE 18
+#define PF_ARM_NEON_INSTRUCTIONS_AVAILABLE 19
+#define PF_SECOND_LEVEL_ADDRESS_TRANSLATION 20
+#define PF_VIRT_FIRMWARE_ENABLED 21
+#define PF_RDWRFSGSBASE_AVAILABLE 22
+#define PF_FASTFAIL_AVAILABLE 23
+#define PF_ARM_DIVIDE_INSTRUCTION_AVAILABLE 24
+#define PF_ARM_64BIT_LOADSTORE_ATOMIC 25
+#define PF_ARM_EXTERNAL_CACHE_AVAILABLE 26
+#define PF_ARM_FMAC_INSTRUCTIONS_AVAILABLE 27
+#define PF_RDRAND_INSTRUCTION_AVAILABLE 28
+#define PF_ARM_V8_INSTRUCTIONS_AVAILABLE 29
+#define PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE 30
+#define PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE 31
#define MAXIMUM_WAIT_OBJECTS 64
KD_OPTION_SET_BLOCK_ENABLE,
} KD_OPTION;
+#ifdef _NTSYSTEM_
typedef VOID
(NTAPI *PKNORMAL_ROUTINE)(
IN PVOID NormalContext OPTIONAL,
IN OUT PVOID *NormalContext OPTIONAL,
IN OUT PVOID *SystemArgument1 OPTIONAL,
IN OUT PVOID *SystemArgument2 OPTIONAL);
+#endif
typedef struct _KAPC {
UCHAR Type;
ULONG SpareLong0;
struct _KTHREAD *Thread;
LIST_ENTRY ApcListEntry;
+#ifdef _NTSYSTEM_
PKKERNEL_ROUTINE KernelRoutine;
PKRUNDOWN_ROUTINE RundownRoutine;
PKNORMAL_ROUTINE NormalRoutine;
+#else
+ PVOID Reserved[3];
+#endif
PVOID NormalContext;
PVOID SystemArgument1;
PVOID SystemArgument2;
typedef PVOID PSECURITY_DESCRIPTOR;
typedef ULONG SECURITY_INFORMATION, *PSECURITY_INFORMATION;
typedef ULONG ACCESS_MASK, *PACCESS_MASK;
+
typedef PVOID PACCESS_TOKEN;
typedef PVOID PSID;
-#define DELETE 0x00010000L
-#define READ_CONTROL 0x00020000L
-#define WRITE_DAC 0x00040000L
-#define WRITE_OWNER 0x00080000L
-#define SYNCHRONIZE 0x00100000L
-#define STANDARD_RIGHTS_REQUIRED 0x000F0000L
-#define STANDARD_RIGHTS_READ READ_CONTROL
-#define STANDARD_RIGHTS_WRITE READ_CONTROL
-#define STANDARD_RIGHTS_EXECUTE READ_CONTROL
-#define STANDARD_RIGHTS_ALL 0x001F0000L
-#define SPECIFIC_RIGHTS_ALL 0x0000FFFFL
-#define ACCESS_SYSTEM_SECURITY 0x01000000L
-#define MAXIMUM_ALLOWED 0x02000000L
-#define GENERIC_READ 0x80000000L
-#define GENERIC_WRITE 0x40000000L
-#define GENERIC_EXECUTE 0x20000000L
-#define GENERIC_ALL 0x10000000L
+#define DELETE 0x00010000L
+#define READ_CONTROL 0x00020000L
+#define WRITE_DAC 0x00040000L
+#define WRITE_OWNER 0x00080000L
+#define SYNCHRONIZE 0x00100000L
+#define STANDARD_RIGHTS_REQUIRED 0x000F0000L
+#define STANDARD_RIGHTS_READ READ_CONTROL
+#define STANDARD_RIGHTS_WRITE READ_CONTROL
+#define STANDARD_RIGHTS_EXECUTE READ_CONTROL
+#define STANDARD_RIGHTS_ALL 0x001F0000L
+#define SPECIFIC_RIGHTS_ALL 0x0000FFFFL
+#define ACCESS_SYSTEM_SECURITY 0x01000000L
+#define MAXIMUM_ALLOWED 0x02000000L
+#define GENERIC_READ 0x80000000L
+#define GENERIC_WRITE 0x40000000L
+#define GENERIC_EXECUTE 0x20000000L
+#define GENERIC_ALL 0x10000000L
typedef struct _GENERIC_MAPPING {
ACCESS_MASK GenericRead;
ACCESS_MASK GenericAll;
} GENERIC_MAPPING, *PGENERIC_MAPPING;
-#define ACL_REVISION 2
-#define ACL_REVISION_DS 4
+#define ACL_REVISION 2
+#define ACL_REVISION_DS 4
-#define ACL_REVISION1 1
-#define ACL_REVISION2 2
-#define ACL_REVISION3 3
-#define ACL_REVISION4 4
-#define MIN_ACL_REVISION ACL_REVISION2
-#define MAX_ACL_REVISION ACL_REVISION4
+#define ACL_REVISION1 1
+#define ACL_REVISION2 2
+#define ACL_REVISION3 3
+#define ACL_REVISION4 4
+#define MIN_ACL_REVISION ACL_REVISION2
+#define MAX_ACL_REVISION ACL_REVISION4
typedef struct _ACL {
UCHAR AclRevision;
ULONG PrivilegeCount;
ULONG Control;
LUID_AND_ATTRIBUTES Privilege[ANYSIZE_ARRAY];
-} PRIVILEGE_SET,*PPRIVILEGE_SET;
+} PRIVILEGE_SET, *PPRIVILEGE_SET;
typedef enum _SECURITY_IMPERSONATION_LEVEL {
SecurityAnonymous,
#define SECURITY_MAX_IMPERSONATION_LEVEL SecurityDelegation
#define SECURITY_MIN_IMPERSONATION_LEVEL SecurityAnonymous
-#define DEFAULT_IMPERSONATION_LEVEL SecurityImpersonation
+#define DEFAULT_IMPERSONATION_LEVEL SecurityImpersonation
#define VALID_IMPERSONATION_LEVEL(Level) (((Level) >= SECURITY_MIN_IMPERSONATION_LEVEL) && ((Level) <= SECURITY_MAX_IMPERSONATION_LEVEL))
#define SECURITY_DYNAMIC_TRACKING (TRUE)
SECURITY_IMPERSONATION_LEVEL Level;
} SE_IMPERSONATION_STATE, *PSE_IMPERSONATION_STATE;
-#define OWNER_SECURITY_INFORMATION (0x00000001L)
-#define GROUP_SECURITY_INFORMATION (0x00000002L)
-#define DACL_SECURITY_INFORMATION (0x00000004L)
-#define SACL_SECURITY_INFORMATION (0x00000008L)
-#define LABEL_SECURITY_INFORMATION (0x00000010L)
-#define PROTECTED_DACL_SECURITY_INFORMATION (0x80000000L)
-#define PROTECTED_SACL_SECURITY_INFORMATION (0x40000000L)
-#define UNPROTECTED_DACL_SECURITY_INFORMATION (0x20000000L)
-#define UNPROTECTED_SACL_SECURITY_INFORMATION (0x10000000L)
+#define OWNER_SECURITY_INFORMATION (0x00000001L)
+#define GROUP_SECURITY_INFORMATION (0x00000002L)
+#define DACL_SECURITY_INFORMATION (0x00000004L)
+#define SACL_SECURITY_INFORMATION (0x00000008L)
+#define LABEL_SECURITY_INFORMATION (0x00000010L)
+
+#define PROTECTED_DACL_SECURITY_INFORMATION (0x80000000L)
+#define PROTECTED_SACL_SECURITY_INFORMATION (0x40000000L)
+#define UNPROTECTED_DACL_SECURITY_INFORMATION (0x20000000L)
+#define UNPROTECTED_SACL_SECURITY_INFORMATION (0x10000000L)
+
typedef enum _SECURITY_OPERATION_CODE {
SetSecurityDescriptor,
#define HIGH_LEVEL 15
#define KI_USER_SHARED_DATA 0xFFFFF78000000000ULL
-#define SharedUserData ((PKUSER_SHARED_DATA const)KI_USER_SHARED_DATA)
+#define SharedUserData ((KUSER_SHARED_DATA * const)KI_USER_SHARED_DATA)
#define SharedInterruptTime (KI_USER_SHARED_DATA + 0x8)
#define SharedSystemTime (KI_USER_SHARED_DATA + 0x14)
#define SharedTickCount (KI_USER_SHARED_DATA + 0x320)
#define EFLAG_ZERO 0x4000
#define EFLAG_SELECT (EFLAG_SIGN | EFLAG_ZERO)
-typedef struct _KFLOATING_SAVE {
- ULONG Dummy;
+typedef struct _KFLOATING_SAVE
+{
+ ULONG Dummy;
} KFLOATING_SAVE, *PKFLOATING_SAVE;
typedef XSAVE_FORMAT XMM_SAVE_AREA32, *PXMM_SAVE_AREA32;
FORCEINLINE
VOID
-KeMemoryBarrier(VOID)
+KeMemoryBarrier(
+ VOID)
{
- // FIXME: Do we really need lfence after the __faststorefence ?
- FastFence();
- LFENCE_ACQUIRE();
+ // FIXME: Do we really need lfence after the __faststorefence ?
+ FastFence();
+ LFENCE_ACQUIRE();
}
#define KeMemoryBarrierWithoutFence() _ReadWriteBarrier()
+_IRQL_requires_max_(HIGH_LEVEL)
+_IRQL_saves_
FORCEINLINE
KIRQL
KeGetCurrentIrql(VOID)
{
- return (KIRQL)__readcr8();
+ return (KIRQL)__readcr8();
}
+_IRQL_requires_max_(HIGH_LEVEL)
FORCEINLINE
VOID
-KeLowerIrql(IN KIRQL NewIrql)
+KeLowerIrql(
+ _In_ _IRQL_restores_ _Notliteral_ KIRQL NewIrql)
{
- //ASSERT((KIRQL)__readcr8() >= NewIrql);
- __writecr8(NewIrql);
+ //ASSERT((KIRQL)__readcr8() >= NewIrql);
+ __writecr8(NewIrql);
}
+_IRQL_requires_max_(HIGH_LEVEL)
+_IRQL_raises_(NewIrql)
+_IRQL_saves_
FORCEINLINE
KIRQL
-KfRaiseIrql(IN KIRQL NewIrql)
+KfRaiseIrql(
+ _In_ KIRQL NewIrql)
{
- KIRQL OldIrql;
+ KIRQL OldIrql;
- OldIrql = (KIRQL)__readcr8();
- //ASSERT(OldIrql <= NewIrql);
- __writecr8(NewIrql);
- return OldIrql;
+ OldIrql = (KIRQL)__readcr8();
+ //ASSERT(OldIrql <= NewIrql);
+ __writecr8(NewIrql);
+ return OldIrql;
}
#define KeRaiseIrql(a,b) *(b) = KfRaiseIrql(a)
+_IRQL_requires_max_(DISPATCH_LEVEL)
+_IRQL_saves_
+_IRQL_raises_(DISPATCH_LEVEL)
FORCEINLINE
KIRQL
-KeRaiseIrqlToDpcLevel(VOID)
+KeRaiseIrqlToDpcLevel(
+ VOID)
{
- return KfRaiseIrql(DISPATCH_LEVEL);
+ return KfRaiseIrql(DISPATCH_LEVEL);
}
FORCEINLINE
KIRQL
KeRaiseIrqlToSynchLevel(VOID)
{
- return KfRaiseIrql(12); // SYNCH_LEVEL = IPI_LEVEL - 2
+ return KfRaiseIrql(12); // SYNCH_LEVEL = IPI_LEVEL - 2
}
FORCEINLINE
PKTHREAD
KeGetCurrentThread(VOID)
{
- return (struct _KTHREAD *)__readgsqword(0x188);
+ return (struct _KTHREAD *)__readgsqword(0x188);
}
+_Always_(_Post_satisfies_(return<=0))
+_Must_inspect_result_
+_IRQL_requires_max_(DISPATCH_LEVEL)
+_Kernel_float_saved_
+_At_(*FloatSave, _Kernel_requires_resource_not_held_(FloatState) _Kernel_acquires_resource_(FloatState))
FORCEINLINE
NTSTATUS
-KeSaveFloatingPointState(PVOID FloatingState)
+KeSaveFloatingPointState(
+ _Out_ PKFLOATING_SAVE FloatSave)
{
- UNREFERENCED_PARAMETER(FloatingState);
- return STATUS_SUCCESS;
+ UNREFERENCED_PARAMETER(FloatSave);
+ return STATUS_SUCCESS;
}
+_Success_(1)
+_Kernel_float_restored_
+_At_(*FloatSave, _Kernel_requires_resource_held_(FloatState) _Kernel_releases_resource_(FloatState))
FORCEINLINE
NTSTATUS
-KeRestoreFloatingPointState(PVOID FloatingState)
+KeRestoreFloatingPointState(
+ _In_ PKFLOATING_SAVE FloatSave)
{
- UNREFERENCED_PARAMETER(FloatingState);
- return STATUS_SUCCESS;
+ UNREFERENCED_PARAMETER(FloatSave);
+ return STATUS_SUCCESS;
}
/* VOID
#elif defined(_M_ARM)
-#include <armddk.h>
-#else
-#error Unknown Architecture
-#endif
+/** Kernel definitions for ARM **/
-NTKERNELAPI
-VOID
+/* Interrupt request levels */
+#define PASSIVE_LEVEL 0
+#define LOW_LEVEL 0
+#define APC_LEVEL 1
+#define DISPATCH_LEVEL 2
+#define CLOCK_LEVEL 13
+#define IPI_LEVEL 14
+#define DRS_LEVEL 14
+#define POWER_LEVEL 14
+#define PROFILE_LEVEL 15
+#define HIGH_LEVEL 15
+
+#define KIP0PCRADDRESS 0xFFDFF000
+#define KI_USER_SHARED_DATA 0xFFFF9000
+#define SharedUserData ((KUSER_SHARED_DATA * const)KI_USER_SHARED_DATA)
+
+#define PAGE_SIZE 0x1000
+#define PAGE_SHIFT 12L
+
+typedef struct _KFLOATING_SAVE
+{
+ ULONG Reserved;
+} KFLOATING_SAVE, *PKFLOATING_SAVE;
+
+extern NTKERNELAPI volatile KSYSTEM_TIME KeTickCount;
+
+FORCEINLINE
+VOID
+YieldProcessor(
+ VOID)
+{
+ __dmb(_ARM_BARRIER_ISHST);
+ __yield();
+}
+
+#define MemoryBarrier() __dmb(_ARM_BARRIER_SY)
+#define PreFetchCacheLine(l,a) __prefetch((const void *) (a))
+#define PrefetchForWrite(p) __prefetch((const void *) (p))
+#define ReadForWriteAccess(p) (*(p))
+
+FORCEINLINE
+VOID
+KeMemoryBarrier(
+ VOID)
+{
+ _ReadWriteBarrier();
+ MemoryBarrier();
+}
+
+#define KeMemoryBarrierWithoutFence() _ReadWriteBarrier()
+
+_IRQL_requires_max_(HIGH_LEVEL)
+_IRQL_saves_
+NTHALAPI
+KIRQL
+NTAPI
+KeGetCurrentIrql(
+ VOID);
+
+_IRQL_requires_max_(HIGH_LEVEL)
+NTHALAPI
+VOID
+FASTCALL
+KfLowerIrql(
+ _In_ _IRQL_restores_ _Notliteral_ KIRQL NewIrql);
+#define KeLowerIrql(a) KfLowerIrql(a)
+
+_IRQL_requires_max_(HIGH_LEVEL)
+_IRQL_raises_(NewIrql)
+_IRQL_saves_
+NTHALAPI
+KIRQL
+FASTCALL
+KfRaiseIrql(
+ _In_ KIRQL NewIrql);
+#define KeRaiseIrql(a,b) *(b) = KfRaiseIrql(a)
+
+_IRQL_requires_max_(DISPATCH_LEVEL)
+_IRQL_saves_
+_IRQL_raises_(DISPATCH_LEVEL)
+NTHALAPI
+KIRQL
+NTAPI
+KeRaiseIrqlToDpcLevel(VOID);
+
+NTHALAPI
+KIRQL
+NTAPI
+KeRaiseIrqlToSynchLevel(VOID);
+
+_Requires_lock_not_held_(*SpinLock)
+_Acquires_lock_(*SpinLock)
+_IRQL_requires_max_(DISPATCH_LEVEL)
+_IRQL_saves_
+_IRQL_raises_(DISPATCH_LEVEL)
+NTHALAPI
+KIRQL
+FASTCALL
+KfAcquireSpinLock(
+ _Inout_ PKSPIN_LOCK SpinLock);
+#define KeAcquireSpinLock(a,b) *(b) = KfAcquireSpinLock(a)
+
+_Requires_lock_held_(*SpinLock)
+_Releases_lock_(*SpinLock)
+_IRQL_requires_(DISPATCH_LEVEL)
+NTHALAPI
+VOID
+FASTCALL
+KfReleaseSpinLock(
+ _Inout_ PKSPIN_LOCK SpinLock,
+ _In_ _IRQL_restores_ KIRQL NewIrql);
+#define KeReleaseSpinLock(a,b) KfReleaseSpinLock(a,b)
+
+_Requires_lock_not_held_(*SpinLock)
+_Acquires_lock_(*SpinLock)
+_IRQL_requires_min_(DISPATCH_LEVEL)
+NTKERNELAPI
+VOID
+FASTCALL
+KefAcquireSpinLockAtDpcLevel(
+ _Inout_ PKSPIN_LOCK SpinLock);
+#define KeAcquireSpinLockAtDpcLevel(SpinLock) KefAcquireSpinLockAtDpcLevel(SpinLock)
+
+_Requires_lock_held_(*SpinLock)
+_Releases_lock_(*SpinLock)
+_IRQL_requires_min_(DISPATCH_LEVEL)
+NTKERNELAPI
+VOID
+FASTCALL
+KefReleaseSpinLockFromDpcLevel(
+ _Inout_ PKSPIN_LOCK SpinLock);
+#define KeReleaseSpinLockFromDpcLevel(SpinLock) KefReleaseSpinLockFromDpcLevel(SpinLock)
+
+NTSYSAPI
+PKTHREAD
+NTAPI
+KeGetCurrentThread(VOID);
+
+_Always_(_Post_satisfies_(return<=0))
+_Must_inspect_result_
+_IRQL_requires_max_(DISPATCH_LEVEL)
+_Kernel_float_saved_
+_At_(*FloatSave, _Kernel_requires_resource_not_held_(FloatState) _Kernel_acquires_resource_(FloatState))
+FORCEINLINE
+NTSTATUS
+KeSaveFloatingPointState(
+ _Out_ PKFLOATING_SAVE FloatSave)
+{
+ UNREFERENCED_PARAMETER(FloatSave);
+ return STATUS_SUCCESS;
+}
+
+_Success_(1)
+_Kernel_float_restored_
+_At_(*FloatSave, _Kernel_requires_resource_held_(FloatState) _Kernel_releases_resource_(FloatState))
+FORCEINLINE
+NTSTATUS
+KeRestoreFloatingPointState(
+ _In_ PKFLOATING_SAVE FloatSave)
+{
+ UNREFERENCED_PARAMETER(FloatSave);
+ return STATUS_SUCCESS;
+}
+
+VOID
+KeFlushIoBuffers(
+ _In_ PMDL Mdl,
+ _In_ BOOLEAN ReadOperation,
+ _In_ BOOLEAN DmaOperation);
+
+#define DbgRaiseAssertionFailure() __emit(0xdefc)
+
+FORCEINLINE
+VOID
+_KeQueryTickCount(
+ OUT PLARGE_INTEGER CurrentCount)
+{
+ for (;;) {
+#ifdef NONAMELESSUNION
+ CurrentCount->s.HighPart = KeTickCount.High1Time;
+ CurrentCount->s.LowPart = KeTickCount.LowPart;
+ if (CurrentCount->s.HighPart == KeTickCount.High2Time) break;
+#else
+ CurrentCount->HighPart = KeTickCount.High1Time;
+ CurrentCount->LowPart = KeTickCount.LowPart;
+ if (CurrentCount->HighPart == KeTickCount.High2Time) break;
+#endif
+ YieldProcessor();
+ }
+}
+#define KeQueryTickCount(CurrentCount) _KeQueryTickCount(CurrentCount)
+
+#define CP15_PMSELR 15, 0, 9, 12, 5 /* Event Counter Selection Register */
+#define CP15_PMXEVCNTR 15, 0, 9, 13, 2 /* Event Count Register */
+#define CP15_TPIDRURW 15, 0, 13, 0, 2 /* Software Thread ID Register, UsRW */
+#define CP15_TPIDRURO 15, 0, 13, 0, 3 /* Software Thread ID Register, UsRO */
+#define CP15_TPIDRPRW 15, 0, 13, 0, 4 /* Software Thread ID Register, Kernel */
+
+#else
+#error Unknown Architecture
+#endif
+
+NTKERNELAPI
+VOID
NTAPI
KeInitializeEvent(
_Out_ PRKEVENT Event,
* Runtime Library Functions *
******************************************************************************/
+#define FAST_FAIL_LEGACY_GS_VIOLATION 0
+#define FAST_FAIL_VTGUARD_CHECK_FAILURE 1
+#define FAST_FAIL_STACK_COOKIE_CHECK_FAILURE 2
+#define FAST_FAIL_CORRUPT_LIST_ENTRY 3
+#define FAST_FAIL_INCORRECT_STACK 4
+#define FAST_FAIL_INVALID_ARG 5
+#define FAST_FAIL_GS_COOKIE_INIT 6
+#define FAST_FAIL_FATAL_APP_EXIT 7
+#define FAST_FAIL_RANGE_CHECK_FAILURE 8
+#define FAST_FAIL_UNSAFE_REGISTRY_ACCESS 9
+#define FAST_FAIL_GUARD_ICALL_CHECK_FAILURE 10
+#define FAST_FAIL_GUARD_WRITE_CHECK_FAILURE 11
+#define FAST_FAIL_INVALID_FIBER_SWITCH 12
+#define FAST_FAIL_INVALID_SET_OF_CONTEXT 13
+#define FAST_FAIL_INVALID_REFERENCE_COUNT 14
+#define FAST_FAIL_INVALID_JUMP_BUFFER 18
+#define FAST_FAIL_MRDATA_MODIFIED 19
+#define FAST_FAIL_INVALID_FAST_FAIL_CODE 0xFFFFFFFF
+
+DECLSPEC_NORETURN
+FORCEINLINE
+VOID
+RtlFailFast(
+ _In_ ULONG Code)
+{
+ __fastfail(Code);
+}
+
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS) && (defined(_M_CEE_PURE) || defined(_M_CEE_SAFE))
+#define NO_KERNEL_LIST_ENTRY_CHECKS
+#endif
+
+#if !defined(EXTRA_KERNEL_LIST_ENTRY_CHECKS) && defined(__REACTOS__)
+#define EXTRA_KERNEL_LIST_ENTRY_CHECKS
+#endif
#if !defined(MIDL_PASS) && !defined(SORTPP_PASS)
return (BOOLEAN)(ListHead->Flink == ListHead);
}
+FORCEINLINE
+BOOLEAN
+RemoveEntryListUnsafe(
+ _In_ PLIST_ENTRY Entry)
+{
+ PLIST_ENTRY OldFlink;
+ PLIST_ENTRY OldBlink;
+
+ OldFlink = Entry->Flink;
+ OldBlink = Entry->Blink;
+ OldFlink->Blink = OldBlink;
+ OldBlink->Flink = OldFlink;
+ return (BOOLEAN)(OldFlink == OldBlink);
+}
+
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+FORCEINLINE
+VOID
+FatalListEntryError(
+ _In_ PVOID P1,
+ _In_ PVOID P2,
+ _In_ PVOID P3)
+{
+ UNREFERENCED_PARAMETER(P1);
+ UNREFERENCED_PARAMETER(P2);
+ UNREFERENCED_PARAMETER(P3);
+
+ RtlFailFast(FAST_FAIL_CORRUPT_LIST_ENTRY);
+}
+
+FORCEINLINE
+VOID
+RtlpCheckListEntry(
+ _In_ PLIST_ENTRY Entry)
+{
+ if (Entry->Flink->Blink != Entry || Entry->Blink->Flink != Entry)
+ FatalListEntryError(Entry->Blink, Entry, Entry->Flink);
+}
+#endif
+
FORCEINLINE
BOOLEAN
RemoveEntryList(
OldFlink = Entry->Flink;
OldBlink = Entry->Blink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+#ifdef EXTRA_KERNEL_LIST_ENTRY_CHECKS
+ if (OldFlink == Entry || OldBlink == Entry)
+ FatalListEntryError(OldBlink, Entry, OldFlink);
+#endif
+ if (OldFlink->Blink != Entry || OldBlink->Flink != Entry)
+ FatalListEntryError(OldBlink, Entry, OldFlink);
+#endif
OldFlink->Blink = OldBlink;
OldBlink->Flink = OldFlink;
return (BOOLEAN)(OldFlink == OldBlink);
PLIST_ENTRY Flink;
PLIST_ENTRY Entry;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS) && DBG
+ RtlpCheckListEntry(ListHead);
+#ifdef EXTRA_KERNEL_LIST_ENTRY_CHECKS
+ if (ListHead->Flink == ListHead || ListHead->Blink == ListHead)
+ FatalListEntryError(ListHead->Blink, ListHead, ListHead->Flink);
+#endif
+#endif
Entry = ListHead->Flink;
Flink = Entry->Flink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+ if (Entry->Blink != ListHead || Flink->Blink != Entry)
+ FatalListEntryError(ListHead, Entry, Flink);
+#endif
ListHead->Flink = Flink;
Flink->Blink = ListHead;
return Entry;
PLIST_ENTRY Blink;
PLIST_ENTRY Entry;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS) && DBG
+ RtlpCheckListEntry(ListHead);
+#ifdef EXTRA_KERNEL_LIST_ENTRY_CHECKS
+ if (ListHead->Flink == ListHead || ListHead->Blink == ListHead)
+ FatalListEntryError(ListHead->Blink, ListHead, ListHead->Flink);
+#endif
+#endif
Entry = ListHead->Blink;
Blink = Entry->Blink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+ if (Blink->Flink != Entry || Entry->Flink != ListHead)
+ FatalListEntryError(Blink, Entry, ListHead);
+#endif
ListHead->Blink = Blink;
Blink->Flink = ListHead;
return Entry;
_Inout_ __drv_aliasesMem PLIST_ENTRY Entry)
{
PLIST_ENTRY OldBlink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS) && DBG
+ RtlpCheckListEntry(ListHead);
+#endif
OldBlink = ListHead->Blink;
Entry->Flink = ListHead;
Entry->Blink = OldBlink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+ if (OldBlink->Flink != ListHead)
+ FatalListEntryError(OldBlink->Blink, OldBlink, ListHead);
+#endif
OldBlink->Flink = Entry;
ListHead->Blink = Entry;
}
_Inout_ __drv_aliasesMem PLIST_ENTRY Entry)
{
PLIST_ENTRY OldFlink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS) && DBG
+ RtlpCheckListEntry(ListHead);
+#endif
OldFlink = ListHead->Flink;
Entry->Flink = OldFlink;
Entry->Blink = ListHead;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+ if (OldFlink->Blink != ListHead)
+ FatalListEntryError(ListHead, OldFlink, OldFlink->Flink);
+#endif
OldFlink->Blink = Entry;
ListHead->Flink = Entry;
}
{
PLIST_ENTRY ListEnd = ListHead->Blink;
+#if !defined(NO_KERNEL_LIST_ENTRY_CHECKS)
+ RtlpCheckListEntry(ListHead);
+ RtlpCheckListEntry(ListToAppend);
+#endif
ListHead->Blink->Flink = ListToAppend;
ListHead->Blink = ListToAppend->Blink;
ListToAppend->Blink->Flink = ListHead;
#endif /* !defined(MIDL_PASS) */
/* Byte Swap Functions */
-#if (defined(_M_IX86) && (_MSC_FULL_VER > 13009037 || defined(__GNUC__))) || \
- ((defined(_M_AMD64) || defined(_M_IA64)) \
- && (_MSC_FULL_VER > 13009175 || defined(__GNUC__)))
-
#define RtlUshortByteSwap(_x) _byteswap_ushort((USHORT)(_x))
#define RtlUlongByteSwap(_x) _byteswap_ulong((_x))
#define RtlUlonglongByteSwap(_x) _byteswap_uint64((_x))
-#endif
-
#if DBG
#define RTL_VERIFY(exp) \
# define __assert_annotationW(msg) __annotation(L"Debug", L"AssertFail", msg)
#else
# define __assert_annotationA(msg) \
- DbgPrint("Assertion %s(%d): %s", __FILE__, __LINE__, msg)
+ DbgPrint("Assertion failed at %s(%d): %s\n", __FILE__, __LINE__, msg)
# define __assert_annotationW(msg) \
- DbgPrint("Assertion %s(%d): %S", __FILE__, __LINE__, msg)
+ DbgPrint("Assertion failed at %s(%d): %S\n", __FILE__, __LINE__, msg)
+#endif
+
+#ifdef _PREFAST_
+#define __analysis_unreachable() __assume(0)
+#else
+#define __analysis_unreachable() ((void)0)
#endif
#define NT_VERIFY(exp) \
#define NT_ASSERT(exp) \
((VOID)((!(exp)) ? \
(__assert_annotationA(#exp), \
- DbgRaiseAssertionFailure(), FALSE) : TRUE))
+ DbgRaiseAssertionFailure(), __analysis_unreachable(), FALSE) : TRUE))
#define NT_ASSERTMSG(msg, exp) \
((VOID)((!(exp)) ? \
(__assert_annotationA(msg), \
- DbgRaiseAssertionFailure(), FALSE) : TRUE))
+ DbgRaiseAssertionFailure(), __analysis_unreachable(), FALSE) : TRUE))
#define NT_ASSERTMSGW(msg, exp) \
((VOID)((!(exp)) ? \
(__assert_annotationW(msg), \
- DbgRaiseAssertionFailure(), FALSE) : TRUE))
+ DbgRaiseAssertionFailure(), __analysis_unreachable(), FALSE) : TRUE))
#else /* !DBG */
#if !defined(_WINBASE_)
-#if defined(_WIN64) && (defined(_NTDRIVER_) || defined(_NTDDK_) || defined(_NTIFS_) || defined(_NTHAL_) || defined(_NTOSP_))
+#if defined(_WIN64) && !defined(_NTSYSTEM_) && (defined(_NTDRIVER_) || defined(_NTDDK_) || defined(_NTIFS_) || defined(_NTHAL_) || !defined(_NTOSP_))
NTKERNELAPI
VOID
InitializeSListHead(
_Out_ PSLIST_HEADER SListHead);
-#else
+#else /* defined(_WIN64) && ... */
+
+/* HACK */
+_IRQL_requires_max_(APC_LEVEL)
+NTKERNELAPI
+DECLSPEC_NORETURN
+VOID
+NTAPI
+ExRaiseStatus(
+ _In_ NTSTATUS Status);
FORCEINLINE
VOID
InitializeSListHead(
_Out_ PSLIST_HEADER SListHead)
{
-#if defined(_IA64_)
- ULONG64 FeatureBits;
-#endif
-
#if defined(_WIN64)
- if (((ULONG_PTR)SListHead & 0xf) != 0) {
- RtlRaiseStatus(STATUS_DATATYPE_MISALIGNMENT);
- }
-#endif
- RtlZeroMemory(SListHead, sizeof(SLIST_HEADER));
+ if (((ULONG_PTR)SListHead & 0xf) != 0) {
+ ExRaiseStatus(STATUS_DATATYPE_MISALIGNMENT);
+ }
#if defined(_IA64_)
- FeatureBits = __getReg(CV_IA64_CPUID4);
- if ((FeatureBits & KF_16BYTE_INSTR) != 0) {
- SListHead->Header16.HeaderType = 1;
- SListHead->Header16.Init = 1;
- }
-#endif
+ SListHead->Region = (ULONG_PTR)SListHead & VRN_MASK;
+#else
+ SListHead->Region = 0;
+#endif /* _IA64_ */
+#endif /* _WIN64 */
+ SListHead->Alignment = 0;
}
-#endif
-
-#if defined(_WIN64)
+#endif /* defined(_WIN64) && ... */
-#define InterlockedPopEntrySList(Head) \
- ExpInterlockedPopEntrySList(Head)
-
-#define InterlockedPushEntrySList(Head, Entry) \
- ExpInterlockedPushEntrySList(Head, Entry)
-
-#define InterlockedFlushSList(Head) \
- ExpInterlockedFlushSList(Head)
-
-#define QueryDepthSList(Head) \
- ExQueryDepthSList(Head)
-
-#else /* !defined(_WIN64) */
+#ifdef _X86_
NTKERNELAPI
PSLIST_ENTRY
FASTCALL
-InterlockedPopEntrySList(
- _Inout_ PSLIST_HEADER ListHead);
+InterlockedPushEntrySList(
+ _Inout_ PSLIST_HEADER SListHead,
+ _Inout_ __drv_aliasesMem PSLIST_ENTRY SListEntry);
NTKERNELAPI
PSLIST_ENTRY
FASTCALL
-InterlockedPushEntrySList(
- _Inout_ PSLIST_HEADER ListHead,
- _Inout_ __drv_aliasesMem PSLIST_ENTRY ListEntry);
+InterlockedPopEntrySList(
+ _Inout_ PSLIST_HEADER SListHead);
+
+#define InterlockedFlushSList(SListHead) \
+ ExInterlockedFlushSList(SListHead)
+
+#else /* !_X86_ */
-#define InterlockedFlushSList(ListHead) \
- ExInterlockedFlushSList(ListHead)
+#define InterlockedPushEntrySList(SListHead, SListEntry) \
+ ExpInterlockedPushEntrySList(SListHead, SListEntry)
-#define QueryDepthSList(Head) \
- ExQueryDepthSList(Head)
+#define InterlockedPopEntrySList(SListHead) \
+ ExpInterlockedPopEntrySList(SListHead)
-#endif /* !defined(_WIN64) */
+#define InterlockedFlushSList(SListHead) \
+ ExpInterlockedFlushSList(SListHead)
+
+#endif /* _X86_ */
+
+#define QueryDepthSList(SListHead) \
+ ExQueryDepthSList(SListHead)
#endif /* !defined(_WINBASE_) */
VOID
NTAPI
MmBuildMdlForNonPagedPool(
- _Inout_ PMDLX MemoryDescriptorList);
+ _Inout_ PMDL MemoryDescriptorList);
//DECLSPEC_DEPRECATED_DDK
NTKERNELAPI
VOID
NTAPI
MmFreePagesFromMdl(
- _Inout_ PMDLX MemoryDescriptorList);
+ _Inout_ PMDL MemoryDescriptorList);
_IRQL_requires_max_(PASSIVE_LEVEL)
NTKERNELAPI
PVOID
NTAPI
MmMapLockedPagesSpecifyCache(
- _Inout_ PMDLX MemoryDescriptorList,
+ _Inout_ PMDL MemoryDescriptorList,
_In_ __drv_strictType(KPROCESSOR_MODE/enum _MODE,__drv_typeConst)
KPROCESSOR_MODE AccessMode,
_In_ __drv_strictTypeMatch(__drv_typeCond) MEMORY_CACHING_TYPE CacheType,
VOID
NTAPI
MmProbeAndLockPages(
- _Inout_ PMDLX MemoryDescriptorList,
+ _Inout_ PMDL MemoryDescriptorList,
_In_ KPROCESSOR_MODE AccessMode,
_In_ LOCK_OPERATION Operation);
VOID
NTAPI
MmUnlockPages(
- _Inout_ PMDLX MemoryDescriptorList);
+ _Inout_ PMDL MemoryDescriptorList);
_IRQL_requires_max_(DISPATCH_LEVEL)
NTKERNELAPI
NTSTATUS
NTAPI
MmAdvanceMdl(
- _Inout_ PMDLX Mdl,
+ _Inout_ PMDL Mdl,
_In_ ULONG NumberOfBytes);
_Must_inspect_result_
MmMapLockedPagesWithReservedMapping(
_In_ PVOID MappingAddress,
_In_ ULONG PoolTag,
- _Inout_ PMDLX MemoryDescriptorList,
+ _Inout_ PMDL MemoryDescriptorList,
_In_ __drv_strictTypeMatch(__drv_typeCond)
MEMORY_CACHING_TYPE CacheType);
NTSTATUS
NTAPI
MmProtectMdlSystemAddress(
- _In_ PMDLX MemoryDescriptorList,
+ _In_ PMDL MemoryDescriptorList,
_In_ ULONG NewProtect);
_IRQL_requires_max_(DISPATCH_LEVEL)
MmUnmapReservedMapping(
_In_ PVOID BaseAddress,
_In_ ULONG PoolTag,
- _Inout_ PMDLX MemoryDescriptorList);
+ _Inout_ PMDL MemoryDescriptorList);
_IRQL_requires_max_ (APC_LEVEL)
NTKERNELAPI
NTSTATUS
NTAPI
IoWMIOpenBlock(
- _In_ GUID *DataBlockGuid,
+ _In_ LPCGUID DataBlockGuid,
_In_ ULONG DesiredAccess,
_Out_ PVOID *DataBlockObject);
#define ExInterlockedPushEntryList ExfInterlockedPushEntryList
#endif /* defined(_X86_) */
-#if defined(_WIN64)
+#ifdef _X86_
+
+#ifdef _WIN2K_COMPAT_SLIST_USAGE
-#if defined(_NTDRIVER_) || defined(_NTDDK_) || defined(_NTIFS_) || \
- defined(_NTHAL_) || defined(_NTOSP_)
NTKERNELAPI
-USHORT
-ExQueryDepthSList(_In_ PSLIST_HEADER ListHead);
-#else
-FORCEINLINE
-USHORT
-ExQueryDepthSList(_In_ PSLIST_HEADER ListHead)
-{
- return (USHORT)(ListHead->Alignment & 0xffff);
-}
-#endif
+PSLIST_ENTRY
+FASTCALL
+ExInterlockedPushEntrySList(
+ _Inout_ PSLIST_HEADER SListHead,
+ _Inout_ __drv_aliasesMem PSLIST_ENTRY SListEntry,
+ _Inout_opt_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
NTKERNELAPI
PSLIST_ENTRY
-ExpInterlockedFlushSList(
- PSLIST_HEADER ListHead);
+FASTCALL
+ExInterlockedPopEntrySList(
+ _Inout_ PSLIST_HEADER SListHead,
+ _Inout_opt_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
+
+#else /* !_WIN2K_COMPAT_SLIST_USAGE */
+
+#define ExInterlockedPushEntrySList(SListHead, SListEntry, Lock) \
+ InterlockedPushEntrySList(SListHead, SListEntry)
+
+#define ExInterlockedPopEntrySList(SListHead, Lock) \
+ InterlockedPopEntrySList(SListHead)
+
+#endif /* _WIN2K_COMPAT_SLIST_USAGE */
NTKERNELAPI
PSLIST_ENTRY
-ExpInterlockedPopEntrySList(
- _Inout_ PSLIST_HEADER ListHead);
+FASTCALL
+ExInterlockedFlushSList(
+ _Inout_ PSLIST_HEADER SListHead);
+
+#ifdef NONAMELESSUNION
+#define ExQueryDepthSList(SListHead) (SListHead)->s.Depth
+#else
+#define ExQueryDepthSList(SListHead) (SListHead)->Depth
+#endif
+
+#else /* !_X86_ */
NTKERNELAPI
PSLIST_ENTRY
ExpInterlockedPushEntrySList(
- _Inout_ PSLIST_HEADER ListHead,
- _Inout_ __drv_aliasesMem PSLIST_ENTRY ListEntry);
+ _Inout_ PSLIST_HEADER SListHead,
+ _Inout_ __drv_aliasesMem PSLIST_ENTRY SListEntry);
-#define ExInterlockedFlushSList(Head) \
- ExpInterlockedFlushSList(Head)
-#define ExInterlockedPopEntrySList(Head, Lock) \
- ExpInterlockedPopEntrySList(Head)
-#define ExInterlockedPushEntrySList(Head, Entry, Lock) \
- ExpInterlockedPushEntrySList(Head, Entry)
+NTKERNELAPI
+PSLIST_ENTRY
+ExpInterlockedPopEntrySList(
+ _Inout_ PSLIST_HEADER SListHead);
-#else /* !defined(_WIN64) */
+NTKERNELAPI
+PSLIST_ENTRY
+ExpInterlockedFlushSList(
+ _Inout_ PSLIST_HEADER SListHead);
-#ifdef NONAMELESSUNION
-#define ExQueryDepthSList(listhead) (listhead)->s.Depth
+#if !defined(_NTSYSTEM_) && (defined(_NTDRIVER_) || defined(_NTDDK_) || defined(_NTIFS_) || defined(_NTHAL_) || defined(_NTOSP_))
+NTKERNELAPI
+USHORT
+ExQueryDepthSList(_In_ PSLIST_HEADER SListHead);
#else
-#define ExQueryDepthSList(listhead) (listhead)->Depth
+FORCEINLINE
+USHORT
+ExQueryDepthSList(_In_ PSLIST_HEADER SListHead)
+{
+#ifdef _WIN64
+ return (USHORT)(SListHead->Alignment & 0xffff);
+#else /* !_WIN64 */
+ return (USHORT)SListHead->Depth;
+#endif /* _WIN64 */
+}
#endif
-NTKERNELAPI
-PSINGLE_LIST_ENTRY
-FASTCALL
-ExInterlockedFlushSList(
- _Inout_ PSLIST_HEADER ListHead);
+#define ExInterlockedPushEntrySList(SListHead, SListEntry, Lock) \
+ ExpInterlockedPushEntrySList(SListHead, SListEntry)
-#endif /* !defined(_WIN64) */
+#define ExInterlockedPopEntrySList(SListHead, Lock) \
+ ExpInterlockedPopEntrySList(SListHead)
-#if defined(_WIN2K_COMPAT_SLIST_USAGE) && defined(_X86_)
+#define ExInterlockedFlushSList(SListHead) \
+ ExpInterlockedFlushSList(SListHead)
-NTKERNELAPI
-PSINGLE_LIST_ENTRY
-FASTCALL
-ExInterlockedPopEntrySList(
- _Inout_ PSLIST_HEADER ListHead,
- _Inout_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
+#endif /* _X86_ */
-NTKERNELAPI
-PSINGLE_LIST_ENTRY
-FASTCALL
-ExInterlockedPushEntrySList(
- _Inout_ PSLIST_HEADER ListHead,
- _Inout_ __drv_aliasesMem PSINGLE_LIST_ENTRY ListEntry,
- _Inout_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
+
+#if defined(_WIN2K_COMPAT_SLIST_USAGE) && defined(_X86_)
_IRQL_requires_max_(APC_LEVEL)
NTKERNELAPI
#else /* !_WIN2K_COMPAT_SLIST_USAGE */
-#if !defined(_WIN64)
-#define ExInterlockedPopEntrySList(_ListHead, _Lock) \
- InterlockedPopEntrySList(_ListHead)
-#define ExInterlockedPushEntrySList(_ListHead, _ListEntry, _Lock) \
- InterlockedPushEntrySList(_ListHead, _ListEntry)
-#endif
-
_IRQL_requires_max_(APC_LEVEL)
static __inline
PVOID
VOID
NTAPI
ExFreePool(
- _In_ __drv_freesMem(Mem) PVOID P);
+ _Pre_notnull_ __drv_freesMem(Mem) PVOID P);
_IRQL_requires_max_(DISPATCH_LEVEL)
NTKERNELAPI
VOID
NTAPI
ExFreePoolWithTag(
- _In_ __drv_freesMem(Mem) PVOID P,
+ _Pre_notnull_ __drv_freesMem(Mem) PVOID P,
_In_ ULONG Tag);
_IRQL_requires_max_(DISPATCH_LEVEL)
_In_ LARGE_INTEGER Increment,
_Inout_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
-#if defined(_WIN64)
+#if !defined(_M_IX86)
#define ExInterlockedAddLargeStatistic(Addend, Increment) \
(VOID)InterlockedAdd64(&(Addend)->QuadPart, Increment)
#else
#define ExInterlockedAddLargeStatistic(Addend, Increment) \
- _InterlockedAddLargeStatistic((PLONGLONG)&(Addend)->QuadPart, Increment)
+ (VOID)_InterlockedAddLargeStatistic((PLONGLONG)&(Addend)->QuadPart, Increment)
#endif
NTKERNELAPI
_In_ ULONG Increment,
_Inout_ _Requires_lock_not_held_(*_Curr_) PKSPIN_LOCK Lock);
-#if defined(_AMD64_) || defined(_IA64_)
-
-#define ExInterlockedCompareExchange64(Destination, Exchange, Comperand, Lock) \
- InterlockedCompareExchange64(Destination, *(Exchange), *(Comperand))
-
-#elif defined(_X86_)
+#if defined(_M_IX86)
NTKERNELAPI
LONGLONG
#else
-NTKERNELAPI
-LONGLONG
-FASTCALL
-ExInterlockedCompareExchange64(
- IN OUT LONGLONG volatile *Destination,
- IN PLONGLONG Exchange,
- IN PLONGLONG Comparand,
- IN PKSPIN_LOCK Lock);
+#define ExInterlockedCompareExchange64(Destination, Exchange, Comperand, Lock) \
+ InterlockedCompareExchange64(Destination, *(Exchange), *(Comperand))
-#endif /* defined(_AMD64_) || defined(_IA64_) */
+#endif /* defined(_M_IX86) */
NTKERNELAPI
PLIST_ENTRY
ExFlushLookasideListEx(
_Inout_ PLOOKASIDE_LIST_EX Lookaside);
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable:__WARNING_MEMORY_NOT_ACQUIRED)
+#endif
+
+__drv_allocatesMem(Mem)
_Must_inspect_result_
_IRQL_requires_max_(DISPATCH_LEVEL)
FORCEINLINE
return Entry;
}
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
_IRQL_requires_max_(DISPATCH_LEVEL)
FORCEINLINE
VOID
ExFreeToLookasideListEx(
_Inout_ PLOOKASIDE_LIST_EX Lookaside,
- _In_ PVOID Entry)
+ _In_ __drv_freesMem(Entry) PVOID Entry)
{
Lookaside->L.TotalFrees += 1;
if (ExQueryDepthSList(&Lookaside->L.ListHead) >= Lookaside->L.Depth) {
#endif /* (NTDDI_VERSION >= NTDDI_WIN7) */
+__drv_allocatesMem(Mem)
_IRQL_requires_max_(DISPATCH_LEVEL)
-static __inline PVOID
+_Ret_maybenull_
+_Post_writable_byte_size_(Lookaside->L.Size)
+static __inline
+PVOID
ExAllocateFromNPagedLookasideList(
_Inout_ PNPAGED_LOOKASIDE_LIST Lookaside)
{
}
_IRQL_requires_max_(DISPATCH_LEVEL)
-static __inline VOID
+static __inline
+VOID
ExFreeToNPagedLookasideList(
_Inout_ PNPAGED_LOOKASIDE_LIST Lookaside,
- _In_ PVOID Entry)
+ _In_ __drv_freesMem(Mem) PVOID Entry)
{
Lookaside->L.TotalFrees++;
#ifdef NONAMELESSUNION
projects / reactos.git / blobdiff