reactos/ntoskrnl/mm/ARM3/miarm.h

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
   5  * PURPOSE:         ARM Memory Manager Header
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
  10
  11 /* Make the code cleaner with some definitions for size multiples */
  12 #define _1KB (1024u)
  13 #define _1MB (1024 * _1KB)
  14 #define _1GB (1024 * _1MB)
  15
  16 /* Everyone loves 64K */
  17 #define _64K (64 * _1KB)
  18
  19 /* Area mapped by a PDE */
  20 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
  21
  22 /* Size of a page table */
  23 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
  24
  25 /* Size of a page directory */
  26 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
  27
  28 /* Stop using these! */
  29 #define PD_COUNT  PPE_PER_PAGE
  30 #define PDE_COUNT PDE_PER_PAGE
  31 #define PTE_COUNT PTE_PER_PAGE
  32
  33 /* Size of all page directories for a process */
  34 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
  35 #ifdef _M_IX86
  36 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
  37 #endif
  38
  39 //
  40 // Protection Bits part of the internal memory manager Protection Mask, from:
  41 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
  42 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
  43 // and public assertions.
  44 //
  45 #define MM_ZERO_ACCESS         0
  46 #define MM_READONLY            1
  47 #define MM_EXECUTE             2
  48 #define MM_EXECUTE_READ        3
  49 #define MM_READWRITE           4
  50 #define MM_WRITECOPY           5
  51 #define MM_EXECUTE_READWRITE   6
  52 #define MM_EXECUTE_WRITECOPY   7
  53 #define MM_PROTECT_ACCESS      7
  54
  55 //
  56 // These are flags on top of the actual protection mask
  57 //
  58 #define MM_NOCACHE            0x08
  59 #define MM_GUARDPAGE          0x10
  60 #define MM_WRITECOMBINE       0x18
  61 #define MM_PROTECT_SPECIAL    0x18
  62
  63 //
  64 // These are special cases
  65 //
  66 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
  67 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
  68 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
  69 #define MM_INVALID_PROTECTION  0xFFFFFFFF
  70
  71 //
  72 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
  73 // The Memory Manager's definition define the attributes that must be preserved
  74 // and these PTE definitions describe the attributes in the hardware sense. This
  75 // helps deal with hardware differences between the actual boolean expression of
  76 // the argument.
  77 //
  78 // For example, in the logical attributes, we want to express read-only as a flag
  79 // but on x86, it is writability that must be set. On the other hand, on x86, just
  80 // like in the kernel, it is disabling the caches that requires a special flag,
  81 // while on certain architectures such as ARM, it is enabling the cache which
  82 // requires a flag.
  83 //
  84 #if defined(_M_IX86) || defined(_M_AMD64)
  85 //
  86 // Access Flags
  87 //
  88 #define PTE_READONLY            0 // Doesn't exist on x86
  89 #define PTE_EXECUTE             0 // Not worrying about NX yet
  90 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
  91 #define PTE_READWRITE           0x2
  92 #define PTE_WRITECOPY           0x200
  93 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
  94 #define PTE_EXECUTE_WRITECOPY   0x200
  95 #define PTE_PROTOTYPE           0x400
  96
  97 //
  98 // State Flags
  99 //
 100 #define PTE_VALID               0x1
 101 #define PTE_ACCESSED            0x20
 102 #define PTE_DIRTY               0x40
 103
 104 //
 105 // Cache flags
 106 //
 107 #define PTE_ENABLE_CACHE        0
 108 #define PTE_DISABLE_CACHE       0x10
 109 #define PTE_WRITECOMBINED_CACHE 0x10
 110 #elif defined(_M_ARM)
 111 #define PTE_READONLY            0x200
 112 #define PTE_EXECUTE             0 // Not worrying about NX yet
 113 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 114 #define PTE_READWRITE           0 // Doesn't exist on ARM
 115 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
 116 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
 117 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
 118 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
 119 //
 120 // Cache flags
 121 //
 122 #define PTE_ENABLE_CACHE        0
 123 #define PTE_DISABLE_CACHE       0x10
 124 #define PTE_WRITECOMBINED_CACHE 0x10
 125 #else
 126 #error Define these please!
 127 #endif
 128
 129 extern const ULONG_PTR MmProtectToPteMask[32];
 130 extern const ULONG MmProtectToValue[32];
 131
 132 //
 133 // Assertions for session images, addresses, and PTEs
 134 //
 135 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
 136     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
 137
 138 #define MI_IS_SESSION_ADDRESS(Address) \
 139     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
 140
 141 #define MI_IS_SESSION_PTE(Pte) \
 142     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
 143
 144 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
 145     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
 146
 147 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
 148     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
 149
 150 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
 151     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
 152
 153 //
 154 // Creates a software PTE with the given protection
 155 //
 156 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
 157
 158 //
 159 // Marks a PTE as deleted
 160 //
 161 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
 162 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
 163
 164 //
 165 // Special values for LoadedImports
 166 //
 167 #ifdef _WIN64
 168 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
 169 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
 170 #else
 171 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
 172 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
 173 #endif
 174 #define MM_SYSLDR_SINGLE_ENTRY 0x1
 175
 176 //
 177 // Number of initial session IDs
 178 //
 179 #define MI_INITIAL_SESSION_IDS  64
 180
 181 #if defined(_M_IX86) || defined(_M_ARM)
 182 //
 183 // PFN List Sentinel
 184 //
 185 #define LIST_HEAD 0xFFFFFFFF
 186
 187 //
 188 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
 189 // we need a manual definition suited to the number of bits in the PteFrame.
 190 // This is used as a LIST_HEAD for the colored list
 191 //
 192 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
 193 #elif defined(_M_AMD64)
 194 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
 195 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
 196 #else
 197 #error Define these please!
 198 #endif
 199
 200 //
 201 // Special IRQL value (found in assertions)
 202 //
 203 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 204
 205 //
 206 // Returns the color of a page
 207 //
 208 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
 209 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
 210 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
 211
 212 //
 213 // Prototype PTEs that don't yet have a pagefile association
 214 //
 215 #ifdef _WIN64
 216 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
 217 #else
 218 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
 219 #endif
 220
 221 //
 222 // Number of session data and tag pages
 223 //
 224 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 225 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 226
 227 //
 228 // Used by MiCheckSecuredVad
 229 //
 230 #define MM_READ_WRITE_ALLOWED   11
 231 #define MM_READ_ONLY_ALLOWED    10
 232 #define MM_NO_ACCESS_ALLOWED    01
 233 #define MM_DELETE_CHECK         85
 234
 235 //
 236 // System views are binned into 64K chunks
 237 //
 238 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
 239
 240 //
 241 // FIXFIX: These should go in ex.h after the pool merge
 242 //
 243 #ifdef _WIN64
 244 #define POOL_BLOCK_SIZE 16
 245 #else
 246 #define POOL_BLOCK_SIZE  8
 247 #endif
 248 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
 249 #define BASE_POOL_TYPE_MASK 1
 250 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
 251
 252 //
 253 // Pool debugging/analysis/tracing flags
 254 //
 255 #define POOL_FLAG_CHECK_TIMERS 0x1
 256 #define POOL_FLAG_CHECK_WORKERS 0x2
 257 #define POOL_FLAG_CHECK_RESOURCES 0x4
 258 #define POOL_FLAG_VERIFIER 0x8
 259 #define POOL_FLAG_CHECK_DEADLOCK 0x10
 260 #define POOL_FLAG_SPECIAL_POOL 0x20
 261 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
 262 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
 263
 264 //
 265 // BAD_POOL_HEADER codes during pool bugcheck
 266 //
 267 #define POOL_CORRUPTED_LIST 3
 268 #define POOL_SIZE_OR_INDEX_MISMATCH 5
 269 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
 270 #define POOL_HEADER_NOT_ALIGNED 7
 271 #define POOL_HEADER_IS_ZERO 8
 272 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
 273 #define POOL_ENTRY_NOT_FOUND 10
 274
 275 //
 276 // BAD_POOL_CALLER codes during pool bugcheck
 277 //
 278 #define POOL_ENTRY_CORRUPTED 1
 279 #define POOL_ENTRY_ALREADY_FREE 6
 280 #define POOL_ENTRY_NOT_ALLOCATED 7
 281 #define POOL_ALLOC_IRQL_INVALID 8
 282 #define POOL_FREE_IRQL_INVALID 9
 283 #define POOL_BILLED_PROCESS_INVALID 13
 284 #define POOL_HEADER_SIZE_INVALID 32
 285
 286 typedef struct _POOL_DESCRIPTOR
 287 {
 288     POOL_TYPE PoolType;
 289     ULONG PoolIndex;
 290     ULONG RunningAllocs;
 291     ULONG RunningDeAllocs;
 292     ULONG TotalPages;
 293     ULONG TotalBigPages;
 294     ULONG Threshold;
 295     PVOID LockAddress;
 296     PVOID PendingFrees;
 297     LONG PendingFreeDepth;
 298     SIZE_T TotalBytes;
 299     SIZE_T Spare0;
 300     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
 301 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 302
 303 typedef struct _POOL_HEADER
 304 {
 305     union
 306     {
 307         struct
 308         {
 309 #ifdef _WIN64
 310             USHORT PreviousSize:8;
 311             USHORT PoolIndex:8;
 312             USHORT BlockSize:8;
 313             USHORT PoolType:8;
 314 #else
 315             USHORT PreviousSize:9;
 316             USHORT PoolIndex:7;
 317             USHORT BlockSize:9;
 318             USHORT PoolType:7;
 319 #endif
 320         };
 321         ULONG Ulong1;
 322     };
 323 #ifdef _WIN64
 324     ULONG PoolTag;
 325 #endif
 326     union
 327     {
 328 #ifdef _WIN64
 329         PEPROCESS ProcessBilled;
 330 #else
 331         ULONG PoolTag;
 332 #endif
 333         struct
 334         {
 335             USHORT AllocatorBackTraceIndex;
 336             USHORT PoolTagHash;
 337         };
 338     };
 339 } POOL_HEADER, *PPOOL_HEADER;
 340
 341 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
 342 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
 343
 344 typedef struct _POOL_TRACKER_TABLE
 345 {
 346     ULONG Key;
 347     LONG NonPagedAllocs;
 348     LONG NonPagedFrees;
 349     SIZE_T NonPagedBytes;
 350     LONG PagedAllocs;
 351     LONG PagedFrees;
 352     SIZE_T PagedBytes;
 353 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
 354
 355 typedef struct _POOL_TRACKER_BIG_PAGES
 356 {
 357     PVOID Va;
 358     ULONG Key;
 359     ULONG NumberOfPages;
 360     PVOID QuotaObject;
 361 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
 362
 363 extern ULONG ExpNumberOfPagedPools;
 364 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
 365 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
 366 extern PPOOL_TRACKER_TABLE PoolTrackTable;
 367
 368 //
 369 // END FIXFIX
 370 //
 371
 372 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
 373 {
 374     LIST_ENTRY Links;
 375     UNICODE_STRING BaseName;
 376 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
 377
 378 typedef enum _MMSYSTEM_PTE_POOL_TYPE
 379 {
 380     SystemPteSpace,
 381     NonPagedPoolExpansion,
 382     MaximumPtePoolTypes
 383 } MMSYSTEM_PTE_POOL_TYPE;
 384
 385 typedef enum _MI_PFN_CACHE_ATTRIBUTE
 386 {
 387     MiNonCached,
 388     MiCached,
 389     MiWriteCombined,
 390     MiNotMapped
 391 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
 392
 393 typedef struct _PHYSICAL_MEMORY_RUN
 394 {
 395     PFN_NUMBER BasePage;
 396     PFN_NUMBER PageCount;
 397 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
 398
 399 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
 400 {
 401     ULONG NumberOfRuns;
 402     PFN_NUMBER NumberOfPages;
 403     PHYSICAL_MEMORY_RUN Run[1];
 404 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
 405
 406 typedef struct _MMCOLOR_TABLES
 407 {
 408     PFN_NUMBER Flink;
 409     PVOID Blink;
 410     PFN_NUMBER Count;
 411 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 412
 413 typedef struct _MI_LARGE_PAGE_RANGES
 414 {
 415     PFN_NUMBER StartFrame;
 416     PFN_NUMBER LastFrame;
 417 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
 418
 419 typedef struct _MMVIEW
 420 {
 421     ULONG_PTR Entry;
 422     PCONTROL_AREA ControlArea;
 423 } MMVIEW, *PMMVIEW;
 424
 425 typedef struct _MMSESSION
 426 {
 427     KGUARDED_MUTEX SystemSpaceViewLock;
 428     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
 429     PCHAR SystemSpaceViewStart;
 430     PMMVIEW SystemSpaceViewTable;
 431     ULONG SystemSpaceHashSize;
 432     ULONG SystemSpaceHashEntries;
 433     ULONG SystemSpaceHashKey;
 434     ULONG BitmapFailures;
 435     PRTL_BITMAP SystemSpaceBitMap;
 436 } MMSESSION, *PMMSESSION;
 437
 438 typedef struct _MM_SESSION_SPACE_FLAGS
 439 {
 440     ULONG Initialized:1;
 441     ULONG DeletePending:1;
 442     ULONG Filler:30;
 443 } MM_SESSION_SPACE_FLAGS;
 444
 445 typedef struct _MM_SESSION_SPACE
 446 {
 447     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
 448     LONG ReferenceCount;
 449     union
 450     {
 451         ULONG LongFlags;
 452         MM_SESSION_SPACE_FLAGS Flags;
 453     } u;
 454     ULONG SessionId;
 455     LIST_ENTRY ProcessList;
 456     LARGE_INTEGER LastProcessSwappedOutTime;
 457     PFN_NUMBER SessionPageDirectoryIndex;
 458     SIZE_T NonPageablePages;
 459     SIZE_T CommittedPages;
 460     PVOID PagedPoolStart;
 461     PVOID PagedPoolEnd;
 462     PMMPDE PagedPoolBasePde;
 463     ULONG Color;
 464     LONG ResidentProcessCount;
 465     ULONG SessionPoolAllocationFailures[4];
 466     LIST_ENTRY ImageList;
 467     LCID LocaleId;
 468     ULONG AttachCount;
 469     KEVENT AttachEvent;
 470     PEPROCESS LastProcess;
 471     LONG ProcessReferenceToSession;
 472     LIST_ENTRY WsListEntry;
 473     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
 474     MMSESSION Session;
 475     KGUARDED_MUTEX PagedPoolMutex;
 476     MM_PAGED_POOL_INFO PagedPoolInfo;
 477     MMSUPPORT Vm;
 478     PMMWSLE Wsle;
 479     PDRIVER_UNLOAD Win32KDriverUnload;
 480     POOL_DESCRIPTOR PagedPool;
 481 #if defined (_M_AMD64)
 482     MMPDE PageDirectory;
 483 #else
 484     PMMPDE PageTables;
 485 #endif
 486 #if defined (_M_AMD64)
 487     PMMPTE SpecialPoolFirstPte;
 488     PMMPTE SpecialPoolLastPte;
 489     PMMPTE NextPdeForSpecialPoolExpansion;
 490     PMMPTE LastPdeForSpecialPoolExpansion;
 491     PFN_NUMBER SpecialPagesInUse;
 492 #endif
 493     LONG ImageLoadingCount;
 494 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
 495
 496 extern PMM_SESSION_SPACE MmSessionSpace;
 497 extern MMPTE HyperTemplatePte;
 498 extern MMPDE ValidKernelPde;
 499 extern MMPTE ValidKernelPte;
 500 extern MMPDE ValidKernelPdeLocal;
 501 extern MMPTE ValidKernelPteLocal;
 502 extern MMPDE DemandZeroPde;
 503 extern MMPTE DemandZeroPte;
 504 extern MMPTE PrototypePte;
 505 extern MMPTE MmDecommittedPte;
 506 extern BOOLEAN MmLargeSystemCache;
 507 extern BOOLEAN MmZeroPageFile;
 508 extern BOOLEAN MmProtectFreedNonPagedPool;
 509 extern BOOLEAN MmTrackLockedPages;
 510 extern BOOLEAN MmTrackPtes;
 511 extern BOOLEAN MmDynamicPfn;
 512 extern BOOLEAN MmMirroring;
 513 extern BOOLEAN MmMakeLowMemory;
 514 extern BOOLEAN MmEnforceWriteProtection;
 515 extern SIZE_T MmAllocationFragment;
 516 extern ULONG MmConsumedPoolPercentage;
 517 extern ULONG MmVerifyDriverBufferType;
 518 extern ULONG MmVerifyDriverLevel;
 519 extern WCHAR MmVerifyDriverBuffer[512];
 520 extern WCHAR MmLargePageDriverBuffer[512];
 521 extern LIST_ENTRY MiLargePageDriverList;
 522 extern BOOLEAN MiLargePageAllDrivers;
 523 extern ULONG MmVerifyDriverBufferLength;
 524 extern ULONG MmLargePageDriverBufferLength;
 525 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
 526 extern SIZE_T MmMaximumNonPagedPoolInBytes;
 527 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
 528 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
 529 extern PVOID MmNonPagedSystemStart;
 530 extern PVOID MmNonPagedPoolStart;
 531 extern PVOID MmNonPagedPoolExpansionStart;
 532 extern PVOID MmNonPagedPoolEnd;
 533 extern SIZE_T MmSizeOfPagedPoolInBytes;
 534 extern PVOID MmPagedPoolStart;
 535 extern PVOID MmPagedPoolEnd;
 536 extern PVOID MmSessionBase;
 537 extern SIZE_T MmSessionSize;
 538 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
 539 extern PMMPTE MiFirstReservedZeroingPte;
 540 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
 541 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
 542 extern SIZE_T MmBootImageSize;
 543 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
 544 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
 545 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
 546 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
 547 extern ULONG_PTR MxPfnAllocation;
 548 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
 549 extern RTL_BITMAP MiPfnBitMap;
 550 extern KGUARDED_MUTEX MmPagedPoolMutex;
 551 extern KGUARDED_MUTEX MmSectionCommitMutex;
 552 extern PVOID MmPagedPoolStart;
 553 extern PVOID MmPagedPoolEnd;
 554 extern PVOID MmNonPagedSystemStart;
 555 extern PVOID MiSystemViewStart;
 556 extern SIZE_T MmSystemViewSize;
 557 extern PVOID MmSessionBase;
 558 extern PVOID MiSessionSpaceEnd;
 559 extern PMMPTE MiSessionImagePteStart;
 560 extern PMMPTE MiSessionImagePteEnd;
 561 extern PMMPTE MiSessionBasePte;
 562 extern PMMPTE MiSessionLastPte;
 563 extern SIZE_T MmSizeOfPagedPoolInBytes;
 564 extern PMMPDE MmSystemPagePtes;
 565 extern PVOID MmSystemCacheStart;
 566 extern PVOID MmSystemCacheEnd;
 567 extern MMSUPPORT MmSystemCacheWs;
 568 extern SIZE_T MmAllocatedNonPagedPool;
 569 extern ULONG MmSpecialPoolTag;
 570 extern PVOID MmHyperSpaceEnd;
 571 extern PMMWSL MmSystemCacheWorkingSetList;
 572 extern SIZE_T MmMinimumNonPagedPoolSize;
 573 extern ULONG MmMinAdditionNonPagedPoolPerMb;
 574 extern SIZE_T MmDefaultMaximumNonPagedPool;
 575 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
 576 extern ULONG MmSecondaryColors;
 577 extern ULONG MmSecondaryColorMask;
 578 extern ULONG MmNumberOfSystemPtes;
 579 extern ULONG MmMaximumNonPagedPoolPercent;
 580 extern ULONG MmLargeStackSize;
 581 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
 582 extern MMPFNLIST MmStandbyPageListByPriority[8];
 583 extern ULONG MmProductType;
 584 extern MM_SYSTEMSIZE MmSystemSize;
 585 extern PKEVENT MiLowMemoryEvent;
 586 extern PKEVENT MiHighMemoryEvent;
 587 extern PKEVENT MiLowPagedPoolEvent;
 588 extern PKEVENT MiHighPagedPoolEvent;
 589 extern PKEVENT MiLowNonPagedPoolEvent;
 590 extern PKEVENT MiHighNonPagedPoolEvent;
 591 extern PFN_NUMBER MmLowMemoryThreshold;
 592 extern PFN_NUMBER MmHighMemoryThreshold;
 593 extern PFN_NUMBER MiLowPagedPoolThreshold;
 594 extern PFN_NUMBER MiHighPagedPoolThreshold;
 595 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
 596 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 597 extern PFN_NUMBER MmMinimumFreePages;
 598 extern PFN_NUMBER MmPlentyFreePages;
 599 extern SIZE_T MmMinimumStackCommitInBytes;
 600 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 601 extern PFN_NUMBER MmResidentAvailablePages;
 602 extern PFN_NUMBER MmResidentAvailableAtInit;
 603 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
 604 extern PFN_NUMBER MmTotalSystemDriverPages;
 605 extern ULONG MmCritsectTimeoutSeconds;
 606 extern PVOID MiSessionImageStart;
 607 extern PVOID MiSessionImageEnd;
 608 extern PMMPTE MiHighestUserPte;
 609 extern PMMPDE MiHighestUserPde;
 610 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
 611 extern PMMPTE MmSharedUserDataPte;
 612 extern LIST_ENTRY MmProcessList;
 613 extern BOOLEAN MmZeroingPageThreadActive;
 614 extern KEVENT MmZeroingPageEvent;
 615 extern ULONG MmSystemPageColor;
 616 extern ULONG MmProcessColorSeed;
 617 extern PMMWSL MmWorkingSetList;
 618 extern PFN_NUMBER MiNumberOfFreePages;
 619 extern SIZE_T MmSessionViewSize;
 620 extern SIZE_T MmSessionPoolSize;
 621 extern SIZE_T MmSessionImageSize;
 622 extern PVOID MiSystemViewStart;
 623 extern PVOID MiSessionPoolEnd;     // 0xBE000000
 624 extern PVOID MiSessionPoolStart;   // 0xBD000000
 625 extern PVOID MiSessionViewStart;   // 0xBE000000
 626 extern PVOID MiSessionSpaceWs;
 627 extern ULONG MmMaximumDeadKernelStacks;
 628 extern SLIST_HEADER MmDeadStackSListHead;
 629 extern MM_AVL_TABLE MmSectionBasedRoot;
 630 extern KGUARDED_MUTEX MmSectionBasedMutex;
 631 extern PVOID MmHighSectionBase;
 632 extern SIZE_T MmSystemLockPagesCount;
 633 extern ULONG_PTR MmSubsectionBase;
 634 extern LARGE_INTEGER MmCriticalSectionTimeout;
 635 extern LIST_ENTRY MmWorkingSetExpansionHead;
 636 extern KSPIN_LOCK MmExpansionLock;
 637 extern PETHREAD MiExpansionLockOwner;
 638
 639 FORCEINLINE
 640 BOOLEAN
 641 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
 642 {
 643     return ((MemoryType == LoaderFree) ||
 644             (MemoryType == LoaderLoadedProgram) ||
 645             (MemoryType == LoaderFirmwareTemporary) ||
 646             (MemoryType == LoaderOsloaderStack));
 647 }
 648
 649 FORCEINLINE
 650 BOOLEAN
 651 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
 652 {
 653     return ((MemoryType == LoaderFirmwarePermanent) ||
 654             (MemoryType == LoaderSpecialMemory) ||
 655             (MemoryType == LoaderHALCachedMemory) ||
 656             (MemoryType == LoaderBBTMemory));
 657 }
 658
 659 #ifdef _M_AMD64
 660 FORCEINLINE
 661 BOOLEAN
 662 MiIsUserPxe(PVOID Address)
 663 {
 664     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
 665 }
 666
 667 FORCEINLINE
 668 BOOLEAN
 669 MiIsUserPpe(PVOID Address)
 670 {
 671     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
 672 }
 673
 674 FORCEINLINE
 675 BOOLEAN
 676 MiIsUserPde(PVOID Address)
 677 {
 678     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
 679 }
 680
 681 FORCEINLINE
 682 BOOLEAN
 683 MiIsUserPte(PVOID Address)
 684 {
 685     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
 686 }
 687 #else
 688 FORCEINLINE
 689 BOOLEAN
 690 MiIsUserPde(PVOID Address)
 691 {
 692     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
 693             (Address <= (PVOID)MiHighestUserPde));
 694 }
 695
 696 FORCEINLINE
 697 BOOLEAN
 698 MiIsUserPte(PVOID Address)
 699 {
 700     return (Address <= (PVOID)MiHighestUserPte);
 701 }
 702 #endif
 703
 704 //
 705 // Figures out the hardware bits for a PTE
 706 //
 707 FORCEINLINE
 708 ULONG_PTR
 709 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
 710 {
 711     MMPTE TempPte;
 712
 713     /* Start fresh */
 714     TempPte.u.Long = 0;
 715
 716     /* Make it valid and accessed */
 717     TempPte.u.Hard.Valid = TRUE;
 718     MI_MAKE_ACCESSED_PAGE(&TempPte);
 719
 720     /* Is this for user-mode? */
 721     if (
 722 #if (_MI_PAGING_LEVELS == 4)
 723         MiIsUserPxe(PointerPte) ||
 724 #endif
 725 #if (_MI_PAGING_LEVELS >= 3)
 726         MiIsUserPpe(PointerPte) ||
 727 #endif
 728         MiIsUserPde(PointerPte) ||
 729         MiIsUserPte(PointerPte))
 730     {
 731         /* Set the owner bit */
 732         MI_MAKE_OWNER_PAGE(&TempPte);
 733     }
 734
 735     /* FIXME: We should also set the global bit */
 736
 737     /* Return the protection */
 738     return TempPte.u.Long;
 739 }
 740
 741 //
 742 // Creates a valid kernel PTE with the given protection
 743 //
 744 FORCEINLINE
 745 VOID
 746 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
 747                             IN PMMPTE MappingPte,
 748                             IN ULONG_PTR ProtectionMask,
 749                             IN PFN_NUMBER PageFrameNumber)
 750 {
 751     /* Only valid for kernel, non-session PTEs */
 752     ASSERT(MappingPte > MiHighestUserPte);
 753     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
 754     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 755
 756     /* Start fresh */
 757     *NewPte = ValidKernelPte;
 758
 759     /* Set the protection and page */
 760     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 761     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 762 }
 763
 764 //
 765 // Creates a valid PTE with the given protection
 766 //
 767 FORCEINLINE
 768 VOID
 769 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
 770                      IN PMMPTE MappingPte,
 771                      IN ULONG_PTR ProtectionMask,
 772                      IN PFN_NUMBER PageFrameNumber)
 773 {
 774     /* Set the protection and page */
 775     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
 776     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 777     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 778 }
 779
 780 //
 781 // Creates a valid user PTE with the given protection
 782 //
 783 FORCEINLINE
 784 VOID
 785 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
 786                           IN PMMPTE MappingPte,
 787                           IN ULONG_PTR ProtectionMask,
 788                           IN PFN_NUMBER PageFrameNumber)
 789 {
 790     /* Only valid for kernel, non-session PTEs */
 791     ASSERT(MappingPte <= MiHighestUserPte);
 792
 793     /* Start fresh */
 794     NewPte->u.Long = 0;
 795
 796     /* Set the protection and page */
 797     NewPte->u.Hard.Valid = TRUE;
 798     NewPte->u.Hard.Owner = TRUE;
 799     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 800     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 801 }
 802
 803 #ifndef _M_AMD64
 804 //
 805 // Builds a Prototype PTE for the address of the PTE
 806 //
 807 FORCEINLINE
 808 VOID
 809 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
 810                       IN PMMPTE PointerPte)
 811 {
 812     ULONG_PTR Offset;
 813
 814     /* Mark this as a prototype */
 815     NewPte->u.Long = 0;
 816     NewPte->u.Proto.Prototype = 1;
 817
 818     /*
 819      * Prototype PTEs are only valid in paged pool by design, this little trick
 820      * lets us only use 30 bits for the adress of the PTE, as long as the area
 821      * stays 1024MB At most.
 822      */
 823     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 824
 825     /*
 826      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
 827      * and the other 21 bits go in the "high"
 828      */
 829     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
 830     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
 831 }
 832
 833 //
 834 // Builds a Subsection PTE for the address of the Segment
 835 //
 836 FORCEINLINE
 837 VOID
 838 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
 839                        IN PVOID Segment)
 840 {
 841     ULONG_PTR Offset;
 842
 843     /* Mark this as a prototype */
 844     NewPte->u.Long = 0;
 845     NewPte->u.Subsect.Prototype = 1;
 846
 847     /*
 848      * Segments are only valid either in nonpaged pool. We store the 20 bit
 849      * difference either from the top or bottom of nonpaged pool, giving a
 850      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
 851      * 256MB.
 852      */
 853     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
 854     {
 855         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
 856         NewPte->u.Subsect.WhichPool = PagedPool;
 857     }
 858     else
 859     {
 860         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
 861         NewPte->u.Subsect.WhichPool = NonPagedPool;
 862     }
 863
 864     /*
 865      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
 866      * and the other 20 bits go in the "high"
 867      */
 868     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
 869     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
 870 }
 871
 872 FORCEINLINE
 873 BOOLEAN
 874 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
 875 {
 876     /// \todo Make this reasonable code, this is UGLY!
 877     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
 878 }
 879
 880 #endif
 881
 882 FORCEINLINE
 883 VOID
 884 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
 885                        _In_ PFN_NUMBER Page,
 886                        _In_ ULONG Protection)
 887 {
 888     NewPte->u.Long = 0;
 889     NewPte->u.Trans.Transition = 1;
 890     NewPte->u.Trans.Protection = Protection;
 891     NewPte->u.Trans.PageFrameNumber = Page;
 892 }
 893
 894 //
 895 // Returns if the page is physically resident (ie: a large page)
 896 // FIXFIX: CISC/x86 only?
 897 //
 898 FORCEINLINE
 899 BOOLEAN
 900 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
 901 {
 902     PMMPDE PointerPde;
 903
 904     /* Large pages are never paged out, always physically resident */
 905     PointerPde = MiAddressToPde(Address);
 906     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
 907 }
 908
 909 //
 910 // Writes a valid PTE
 911 //
 912 FORCEINLINE
 913 VOID
 914 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
 915                    IN MMPTE TempPte)
 916 {
 917     /* Write the valid PTE */
 918     ASSERT(PointerPte->u.Hard.Valid == 0);
 919     ASSERT(TempPte.u.Hard.Valid == 1);
 920     *PointerPte = TempPte;
 921 }
 922
 923 //
 924 // Updates a valid PTE
 925 //
 926 FORCEINLINE
 927 VOID
 928 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
 929                    IN MMPTE TempPte)
 930 {
 931     /* Write the valid PTE */
 932     ASSERT(PointerPte->u.Hard.Valid == 1);
 933     ASSERT(TempPte.u.Hard.Valid == 1);
 934     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
 935     *PointerPte = TempPte;
 936 }
 937
 938 //
 939 // Writes an invalid PTE
 940 //
 941 FORCEINLINE
 942 VOID
 943 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
 944                      IN MMPTE InvalidPte)
 945 {
 946     /* Write the invalid PTE */
 947     ASSERT(InvalidPte.u.Hard.Valid == 0);
 948     ASSERT(InvalidPte.u.Long != 0);
 949     *PointerPte = InvalidPte;
 950 }
 951
 952 //
 953 // Erase the PTE completely
 954 //
 955 FORCEINLINE
 956 VOID
 957 MI_ERASE_PTE(IN PMMPTE PointerPte)
 958 {
 959     /* Zero out the PTE */
 960     ASSERT(PointerPte->u.Long != 0);
 961     PointerPte->u.Long = 0;
 962 }
 963
 964 //
 965 // Writes a valid PDE
 966 //
 967 FORCEINLINE
 968 VOID
 969 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
 970                    IN MMPDE TempPde)
 971 {
 972     /* Write the valid PDE */
 973     ASSERT(PointerPde->u.Hard.Valid == 0);
 974     ASSERT(TempPde.u.Hard.Valid == 1);
 975     *PointerPde = TempPde;
 976 }
 977
 978 //
 979 // Writes an invalid PDE
 980 //
 981 FORCEINLINE
 982 VOID
 983 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
 984                      IN MMPDE InvalidPde)
 985 {
 986     /* Write the invalid PDE */
 987     ASSERT(InvalidPde.u.Hard.Valid == 0);
 988     ASSERT(InvalidPde.u.Long != 0);
 989     *PointerPde = InvalidPde;
 990 }
 991
 992 //
 993 // Checks if the thread already owns a working set
 994 //
 995 FORCEINLINE
 996 BOOLEAN
 997 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
 998 {
 999     /* If any of these are held, return TRUE */
1000     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1001             (Thread->OwnsProcessWorkingSetShared) ||
1002             (Thread->OwnsSystemWorkingSetExclusive) ||
1003             (Thread->OwnsSystemWorkingSetShared) ||
1004             (Thread->OwnsSessionWorkingSetExclusive) ||
1005             (Thread->OwnsSessionWorkingSetShared));
1006 }
1007
1008 //
1009 // Checks if the process owns the working set lock
1010 //
1011 FORCEINLINE
1012 BOOLEAN
1013 MI_WS_OWNER(IN PEPROCESS Process)
1014 {
1015     /* Check if this process is the owner, and that the thread owns the WS */
1016     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1017     {
1018         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1019     }
1020     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1021     {
1022         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1023     }
1024     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1025             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1026              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1027 }
1028
1029 //
1030 // New ARM3<->RosMM PAGE Architecture
1031 //
1032 FORCEINLINE
1033 BOOLEAN
1034 MiIsRosSectionObject(IN PVOID Section)
1035 {
1036     PROS_SECTION_OBJECT RosSection = Section;
1037     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1038     return FALSE;
1039 }
1040
1041 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1042
1043 VOID
1044 NTAPI
1045 MiDecrementReferenceCount(
1046     IN PMMPFN Pfn1,
1047     IN PFN_NUMBER PageFrameIndex
1048 );
1049
1050 FORCEINLINE
1051 BOOLEAN
1052 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1053 {
1054     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1055 }
1056
1057 //
1058 // Locks the working set for the given process
1059 //
1060 FORCEINLINE
1061 VOID
1062 MiLockProcessWorkingSet(IN PEPROCESS Process,
1063                         IN PETHREAD Thread)
1064 {
1065     /* Shouldn't already be owning the process working set */
1066     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1067     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1068
1069     /* Block APCs, make sure that still nothing is already held */
1070     KeEnterGuardedRegion();
1071     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1072
1073     /* Lock the working set */
1074     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1075
1076     /* Now claim that we own the lock */
1077     ASSERT(!MI_IS_WS_UNSAFE(Process));
1078     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1079     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1080 }
1081
1082 FORCEINLINE
1083 VOID
1084 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1085                               IN PETHREAD Thread)
1086 {
1087     /* Shouldn't already be owning the process working set */
1088     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1089     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1090
1091     /* Block APCs, make sure that still nothing is already held */
1092     KeEnterGuardedRegion();
1093     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1094
1095     /* Lock the working set */
1096     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1097
1098     /* Now claim that we own the lock */
1099     ASSERT(!MI_IS_WS_UNSAFE(Process));
1100     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1101     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1102     Thread->OwnsProcessWorkingSetShared = TRUE;
1103 }
1104
1105 FORCEINLINE
1106 VOID
1107 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1108                               IN PETHREAD Thread)
1109 {
1110     /* Shouldn't already be owning the process working set */
1111     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1112
1113     /* APCs must be blocked, make sure that still nothing is already held */
1114     ASSERT(KeAreAllApcsDisabled() == TRUE);
1115     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1116
1117     /* Lock the working set */
1118     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1119
1120     /* Now claim that we own the lock */
1121     ASSERT(!MI_IS_WS_UNSAFE(Process));
1122     Process->Vm.Flags.AcquiredUnsafe = 1;
1123     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1124     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1125 }
1126
1127 //
1128 // Unlocks the working set for the given process
1129 //
1130 FORCEINLINE
1131 VOID
1132 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1133                           IN PETHREAD Thread)
1134 {
1135     /* Make sure we are the owner of a safe acquisition */
1136     ASSERT(MI_WS_OWNER(Process));
1137     ASSERT(!MI_IS_WS_UNSAFE(Process));
1138
1139     /* The thread doesn't own it anymore */
1140     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1141     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1142
1143     /* Release the lock and re-enable APCs */
1144     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1145     KeLeaveGuardedRegion();
1146 }
1147
1148 //
1149 // Unlocks the working set for the given process
1150 //
1151 FORCEINLINE
1152 VOID
1153 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1154                                 IN PETHREAD Thread)
1155 {
1156     /* Make sure we are the owner of a safe acquisition (because shared) */
1157     ASSERT(MI_WS_OWNER(Process));
1158     ASSERT(!MI_IS_WS_UNSAFE(Process));
1159
1160     /* Ensure we are in a shared acquisition */
1161     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1162     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1163
1164     /* Don't claim the lock anylonger */
1165     Thread->OwnsProcessWorkingSetShared = FALSE;
1166
1167     /* Release the lock and re-enable APCs */
1168     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1169     KeLeaveGuardedRegion();
1170 }
1171
1172 //
1173 // Unlocks the working set for the given process
1174 //
1175 FORCEINLINE
1176 VOID
1177 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1178                                 IN PETHREAD Thread)
1179 {
1180     /* Make sure we are the owner of an unsafe acquisition */
1181     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1182     ASSERT(KeAreAllApcsDisabled() == TRUE);
1183     ASSERT(MI_WS_OWNER(Process));
1184     ASSERT(MI_IS_WS_UNSAFE(Process));
1185
1186     /* No longer unsafe */
1187     Process->Vm.Flags.AcquiredUnsafe = 0;
1188
1189     /* The thread doesn't own it anymore */
1190     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1191     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1192
1193     /* Release the lock but don't touch APC state */
1194     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1195     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1196 }
1197
1198 //
1199 // Locks the working set
1200 //
1201 FORCEINLINE
1202 VOID
1203 MiLockWorkingSet(IN PETHREAD Thread,
1204                  IN PMMSUPPORT WorkingSet)
1205 {
1206     /* Block APCs */
1207     KeEnterGuardedRegion();
1208
1209     /* Working set should be in global memory */
1210     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1211
1212     /* Thread shouldn't already be owning something */
1213     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1214
1215     /* Lock this working set */
1216     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1217
1218     /* Which working set is this? */
1219     if (WorkingSet == &MmSystemCacheWs)
1220     {
1221         /* Own the system working set */
1222         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1223                (Thread->OwnsSystemWorkingSetShared == FALSE));
1224         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1225     }
1226     else if (WorkingSet->Flags.SessionSpace)
1227     {
1228         /* Own the session working set */
1229         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1230                (Thread->OwnsSessionWorkingSetShared == FALSE));
1231         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1232     }
1233     else
1234     {
1235         /* Own the process working set */
1236         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1237                (Thread->OwnsProcessWorkingSetShared == FALSE));
1238         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1239     }
1240 }
1241
1242 //
1243 // Unlocks the working set
1244 //
1245 FORCEINLINE
1246 VOID
1247 MiUnlockWorkingSet(IN PETHREAD Thread,
1248                    IN PMMSUPPORT WorkingSet)
1249 {
1250     /* Working set should be in global memory */
1251     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1252
1253     /* Which working set is this? */
1254     if (WorkingSet == &MmSystemCacheWs)
1255     {
1256         /* Release the system working set */
1257         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1258                (Thread->OwnsSystemWorkingSetShared == TRUE));
1259         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1260     }
1261     else if (WorkingSet->Flags.SessionSpace)
1262     {
1263         /* Release the session working set */
1264         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1265                (Thread->OwnsSessionWorkingSetShared == TRUE));
1266         Thread->OwnsSessionWorkingSetExclusive = 0;
1267     }
1268     else
1269     {
1270         /* Release the process working set */
1271         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1272                (Thread->OwnsProcessWorkingSetShared));
1273         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1274     }
1275
1276     /* Release the working set lock */
1277     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1278
1279     /* Unblock APCs */
1280     KeLeaveGuardedRegion();
1281 }
1282
1283 FORCEINLINE
1284 VOID
1285 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1286                                   IN PETHREAD Thread,
1287                                   OUT PBOOLEAN Safe,
1288                                   OUT PBOOLEAN Shared)
1289 {
1290     ASSERT(MI_WS_OWNER(Process));
1291
1292     /* Check if the current owner is unsafe */
1293     if (MI_IS_WS_UNSAFE(Process))
1294     {
1295         /* Release unsafely */
1296         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1297         *Safe = FALSE;
1298         *Shared = FALSE;
1299     }
1300     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1301     {
1302         /* Owner is safe and exclusive, release normally */
1303         MiUnlockProcessWorkingSet(Process, Thread);
1304         *Safe = TRUE;
1305         *Shared = FALSE;
1306     }
1307     else
1308     {
1309         /* Owner is shared (implies safe), release normally */
1310         MiUnlockProcessWorkingSetShared(Process, Thread);
1311         *Safe = TRUE;
1312         *Shared = TRUE;
1313     }
1314 }
1315
1316 FORCEINLINE
1317 VOID
1318 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1319                                 IN PETHREAD Thread,
1320                                 IN BOOLEAN Safe,
1321                                 IN BOOLEAN Shared)
1322 {
1323     /* Check if this was a safe lock or not */
1324     if (Safe)
1325     {
1326         if (Shared)
1327         {
1328             /* Reacquire safely & shared */
1329             MiLockProcessWorkingSetShared(Process, Thread);
1330         }
1331         else
1332         {
1333             /* Reacquire safely */
1334             MiLockProcessWorkingSet(Process, Thread);
1335         }
1336     }
1337     else
1338     {
1339         /* Unsafe lock cannot be shared */
1340         ASSERT(Shared == FALSE);
1341         /* Reacquire unsafely */
1342         MiLockProcessWorkingSetUnsafe(Process, Thread);
1343     }
1344 }
1345
1346 FORCEINLINE
1347 KIRQL
1348 MiAcquireExpansionLock(VOID)
1349 {
1350     KIRQL OldIrql;
1351
1352     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1353     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1354     ASSERT(MiExpansionLockOwner == NULL);
1355     MiExpansionLockOwner = PsGetCurrentThread();
1356     return OldIrql;
1357 }
1358
1359 FORCEINLINE
1360 VOID
1361 MiReleaseExpansionLock(KIRQL OldIrql)
1362 {
1363     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1364     MiExpansionLockOwner = NULL;
1365     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1366     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1367 }
1368
1369 //
1370 // Returns the ProtoPTE inside a VAD for the given VPN
1371 //
1372 FORCEINLINE
1373 PMMPTE
1374 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1375                              IN ULONG_PTR Vpn)
1376 {
1377     PMMPTE ProtoPte;
1378
1379     /* Find the offset within the VAD's prototype PTEs */
1380     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1381     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1382     return ProtoPte;
1383 }
1384
1385 //
1386 // Returns the PFN Database entry for the given page number
1387 // Warning: This is not necessarily a valid PFN database entry!
1388 //
1389 FORCEINLINE
1390 PMMPFN
1391 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1392 {
1393     /* Get the entry */
1394     return &MmPfnDatabase[Pfn];
1395 };
1396
1397 //
1398 // Drops a locked page without dereferencing it
1399 //
1400 FORCEINLINE
1401 VOID
1402 MiDropLockCount(IN PMMPFN Pfn1)
1403 {
1404     /* This page shouldn't be locked, but it should be valid */
1405     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1406     ASSERT(Pfn1->u2.ShareCount == 0);
1407
1408     /* Is this the last reference to the page */
1409     if (Pfn1->u3.e2.ReferenceCount == 1)
1410     {
1411         /* It better not be valid */
1412         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1413
1414         /* Is it a prototype PTE? */
1415         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1416             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1417         {
1418             /* FIXME: We should return commit */
1419             DPRINT1("Not returning commit for prototype PTE\n");
1420         }
1421
1422         /* Update the counter */
1423         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1424     }
1425 }
1426
1427 //
1428 // Drops a locked page and dereferences it
1429 //
1430 FORCEINLINE
1431 VOID
1432 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1433 {
1434     USHORT RefCount, OldRefCount;
1435     PFN_NUMBER PageFrameIndex;
1436
1437     /* Loop while we decrement the page successfully */
1438     do
1439     {
1440         /* There should be at least one reference */
1441         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1442         ASSERT(OldRefCount != 0);
1443
1444         /* Are we the last one */
1445         if (OldRefCount == 1)
1446         {
1447             /* The page shoudln't be shared not active at this point */
1448             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1449             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1450             ASSERT(Pfn1->u2.ShareCount == 0);
1451
1452             /* Is it a prototype PTE? */
1453             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1454                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1455             {
1456                 /* FIXME: We should return commit */
1457                 DPRINT1("Not returning commit for prototype PTE\n");
1458             }
1459
1460             /* Update the counter, and drop a reference the long way */
1461             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1462             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1463             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1464             return;
1465         }
1466
1467         /* Drop a reference the short way, and that's it */
1468         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1469                                                 OldRefCount - 1,
1470                                                 OldRefCount);
1471         ASSERT(RefCount != 0);
1472     } while (OldRefCount != RefCount);
1473
1474     /* If we got here, there should be more than one reference */
1475     ASSERT(RefCount > 1);
1476     if (RefCount == 2)
1477     {
1478         /* Is it still being shared? */
1479         if (Pfn1->u2.ShareCount >= 1)
1480         {
1481             /* Then it should be valid */
1482             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1483
1484             /* Is it a prototype PTE? */
1485             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1486                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1487             {
1488                 /* We don't handle ethis */
1489                 ASSERT(FALSE);
1490             }
1491
1492             /* Update the counter */
1493             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1494         }
1495     }
1496 }
1497
1498 //
1499 // References a locked page and updates the counter
1500 // Used in MmProbeAndLockPages to handle different edge cases
1501 //
1502 FORCEINLINE
1503 VOID
1504 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1505 {
1506     USHORT RefCount, OldRefCount;
1507
1508     /* Sanity check */
1509     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1510
1511     /* Does ARM3 own the page? */
1512     if (MI_IS_ROS_PFN(Pfn1))
1513     {
1514         /* ReactOS Mm doesn't track share count */
1515         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1516     }
1517     else
1518     {
1519         /* On ARM3 pages, we should see a valid share count */
1520         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1521
1522         /* Is it a prototype PTE? */
1523         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1524             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1525         {
1526             /* FIXME: We should charge commit */
1527             DPRINT1("Not charging commit for prototype PTE\n");
1528         }
1529     }
1530
1531     /* More locked pages! */
1532     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1533
1534     /* Loop trying to update the reference count */
1535     do
1536     {
1537         /* Get the current reference count, make sure it's valid */
1538         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1539         ASSERT(OldRefCount != 0);
1540         ASSERT(OldRefCount < 2500);
1541
1542         /* Bump it up by one */
1543         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1544                                                 OldRefCount + 1,
1545                                                 OldRefCount);
1546         ASSERT(RefCount != 0);
1547     } while (OldRefCount != RefCount);
1548
1549     /* Was this the first lock attempt? If not, undo our bump */
1550     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1551 }
1552
1553 //
1554 // References a locked page and updates the counter
1555 // Used in all other cases except MmProbeAndLockPages
1556 //
1557 FORCEINLINE
1558 VOID
1559 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1560 {
1561     USHORT NewRefCount;
1562
1563     /* Is it a prototype PTE? */
1564     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1565         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1566     {
1567         /* FIXME: We should charge commit */
1568         DPRINT1("Not charging commit for prototype PTE\n");
1569     }
1570
1571     /* More locked pages! */
1572     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1573
1574     /* Update the reference count */
1575     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1576     if (NewRefCount == 2)
1577     {
1578         /* Is it locked or shared? */
1579         if (Pfn1->u2.ShareCount)
1580         {
1581             /* It's shared, so make sure it's active */
1582             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1583         }
1584         else
1585         {
1586             /* It's locked, so we shouldn't lock again */
1587             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1588         }
1589     }
1590     else
1591     {
1592         /* Someone had already locked the page, so undo our bump */
1593         ASSERT(NewRefCount < 2500);
1594         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1595     }
1596 }
1597
1598 //
1599 // References a locked page and updates the counter
1600 // Used in all other cases except MmProbeAndLockPages
1601 //
1602 FORCEINLINE
1603 VOID
1604 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1605 {
1606     USHORT NewRefCount;
1607
1608     /* Make sure the page isn't used yet */
1609     ASSERT(Pfn1->u2.ShareCount == 0);
1610     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1611
1612     /* Is it a prototype PTE? */
1613     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1614         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1615     {
1616         /* FIXME: We should charge commit */
1617         DPRINT1("Not charging commit for prototype PTE\n");
1618     }
1619
1620     /* More locked pages! */
1621     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1622
1623     /* Update the reference count */
1624     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1625     if (NewRefCount != 1)
1626     {
1627         /* Someone had already locked the page, so undo our bump */
1628         ASSERT(NewRefCount < 2500);
1629         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1630     }
1631 }
1632
1633 FORCEINLINE
1634 VOID
1635 MiIncrementPageTableReferences(IN PVOID Address)
1636 {
1637     PUSHORT RefCount;
1638
1639     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1640
1641     *RefCount += 1;
1642     ASSERT(*RefCount <= PTE_PER_PAGE);
1643 }
1644
1645 FORCEINLINE
1646 VOID
1647 MiDecrementPageTableReferences(IN PVOID Address)
1648 {
1649     PUSHORT RefCount;
1650
1651     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1652
1653     *RefCount -= 1;
1654     ASSERT(*RefCount < PTE_PER_PAGE);
1655 }
1656
1657 FORCEINLINE
1658 USHORT
1659 MiQueryPageTableReferences(IN PVOID Address)
1660 {
1661     PUSHORT RefCount;
1662
1663     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1664
1665     return *RefCount;
1666 }
1667
1668 BOOLEAN
1669 NTAPI
1670 MmArmInitSystem(
1671     IN ULONG Phase,
1672     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1673 );
1674
1675 VOID
1676 NTAPI
1677 MiInitializeSessionSpaceLayout(VOID);
1678
1679 NTSTATUS
1680 NTAPI
1681 MiInitMachineDependent(
1682     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1683 );
1684
1685 VOID
1686 NTAPI
1687 MiComputeColorInformation(
1688     VOID
1689 );
1690
1691 VOID
1692 NTAPI
1693 MiMapPfnDatabase(
1694     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1695 );
1696
1697 VOID
1698 NTAPI
1699 MiInitializeColorTables(
1700     VOID
1701 );
1702
1703 VOID
1704 NTAPI
1705 MiInitializePfnDatabase(
1706     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1707 );
1708
1709 VOID
1710 NTAPI
1711 MiInitializeSessionWsSupport(
1712     VOID
1713 );
1714
1715 VOID
1716 NTAPI
1717 MiInitializeSessionIds(
1718     VOID
1719 );
1720
1721 BOOLEAN
1722 NTAPI
1723 MiInitializeMemoryEvents(
1724     VOID
1725 );
1726
1727 PFN_NUMBER
1728 NTAPI
1729 MxGetNextPage(
1730     IN PFN_NUMBER PageCount
1731 );
1732
1733 PPHYSICAL_MEMORY_DESCRIPTOR
1734 NTAPI
1735 MmInitializeMemoryLimits(
1736     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1737     IN PBOOLEAN IncludeType
1738 );
1739
1740 PFN_NUMBER
1741 NTAPI
1742 MiPagesInLoaderBlock(
1743     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1744     IN PBOOLEAN IncludeType
1745 );
1746
1747 VOID
1748 FASTCALL
1749 MiSyncARM3WithROS(
1750     IN PVOID AddressStart,
1751     IN PVOID AddressEnd
1752 );
1753
1754 NTSTATUS
1755 NTAPI
1756 MiRosProtectVirtualMemory(
1757     IN PEPROCESS Process,
1758     IN OUT PVOID *BaseAddress,
1759     IN OUT PSIZE_T NumberOfBytesToProtect,
1760     IN ULONG NewAccessProtection,
1761     OUT PULONG OldAccessProtection OPTIONAL
1762 );
1763
1764 NTSTATUS
1765 NTAPI
1766 MmArmAccessFault(
1767     IN BOOLEAN StoreInstruction,
1768     IN PVOID Address,
1769     IN KPROCESSOR_MODE Mode,
1770     IN PVOID TrapInformation
1771 );
1772
1773 NTSTATUS
1774 FASTCALL
1775 MiCheckPdeForPagedPool(
1776     IN PVOID Address
1777 );
1778
1779 VOID
1780 NTAPI
1781 MiInitializeNonPagedPool(
1782     VOID
1783 );
1784
1785 VOID
1786 NTAPI
1787 MiInitializeNonPagedPoolThresholds(
1788     VOID
1789 );
1790
1791 VOID
1792 NTAPI
1793 MiInitializePoolEvents(
1794     VOID
1795 );
1796
1797 VOID                      //
1798 NTAPI                     //
1799 InitializePool(           //
1800     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1801     IN ULONG Threshold    //
1802 );                        //
1803
1804 // FIXFIX: THIS ONE TOO
1805 VOID
1806 NTAPI
1807 INIT_FUNCTION
1808 ExInitializePoolDescriptor(
1809     IN PPOOL_DESCRIPTOR PoolDescriptor,
1810     IN POOL_TYPE PoolType,
1811     IN ULONG PoolIndex,
1812     IN ULONG Threshold,
1813     IN PVOID PoolLock
1814 );
1815
1816 NTSTATUS
1817 NTAPI
1818 MiInitializeSessionPool(
1819     VOID
1820 );
1821
1822 VOID
1823 NTAPI
1824 MiInitializeSystemPtes(
1825     IN PMMPTE StartingPte,
1826     IN ULONG NumberOfPtes,
1827     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1828 );
1829
1830 PMMPTE
1831 NTAPI
1832 MiReserveSystemPtes(
1833     IN ULONG NumberOfPtes,
1834     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1835 );
1836
1837 VOID
1838 NTAPI
1839 MiReleaseSystemPtes(
1840     IN PMMPTE StartingPte,
1841     IN ULONG NumberOfPtes,
1842     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1843 );
1844
1845
1846 PFN_NUMBER
1847 NTAPI
1848 MiFindContiguousPages(
1849     IN PFN_NUMBER LowestPfn,
1850     IN PFN_NUMBER HighestPfn,
1851     IN PFN_NUMBER BoundaryPfn,
1852     IN PFN_NUMBER SizeInPages,
1853     IN MEMORY_CACHING_TYPE CacheType
1854 );
1855
1856 PVOID
1857 NTAPI
1858 MiCheckForContiguousMemory(
1859     IN PVOID BaseAddress,
1860     IN PFN_NUMBER BaseAddressPages,
1861     IN PFN_NUMBER SizeInPages,
1862     IN PFN_NUMBER LowestPfn,
1863     IN PFN_NUMBER HighestPfn,
1864     IN PFN_NUMBER BoundaryPfn,
1865     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1866 );
1867
1868 PMDL
1869 NTAPI
1870 MiAllocatePagesForMdl(
1871     IN PHYSICAL_ADDRESS LowAddress,
1872     IN PHYSICAL_ADDRESS HighAddress,
1873     IN PHYSICAL_ADDRESS SkipBytes,
1874     IN SIZE_T TotalBytes,
1875     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1876     IN ULONG Flags
1877 );
1878
1879 VOID
1880 NTAPI
1881 MiInsertPageInList(
1882     IN PMMPFNLIST ListHead,
1883     IN PFN_NUMBER PageFrameIndex
1884 );
1885
1886 VOID
1887 NTAPI
1888 MiUnlinkFreeOrZeroedPage(
1889     IN PMMPFN Entry
1890 );
1891
1892 VOID
1893 NTAPI
1894 MiUnlinkPageFromList(
1895     IN PMMPFN Pfn
1896 );
1897
1898 VOID
1899 NTAPI
1900 MiInitializePfn(
1901     IN PFN_NUMBER PageFrameIndex,
1902     IN PMMPTE PointerPte,
1903     IN BOOLEAN Modified
1904 );
1905
1906 NTSTATUS
1907 NTAPI
1908 MiInitializeAndChargePfn(
1909     OUT PPFN_NUMBER PageFrameIndex,
1910     IN PMMPDE PointerPde,
1911     IN PFN_NUMBER ContainingPageFrame,
1912     IN BOOLEAN SessionAllocation
1913 );
1914
1915 VOID
1916 NTAPI
1917 MiInitializePfnAndMakePteValid(
1918     IN PFN_NUMBER PageFrameIndex,
1919     IN PMMPTE PointerPte,
1920     IN MMPTE TempPte
1921 );
1922
1923 VOID
1924 NTAPI
1925 MiInitializePfnForOtherProcess(
1926     IN PFN_NUMBER PageFrameIndex,
1927     IN PVOID PteAddress,
1928     IN PFN_NUMBER PteFrame
1929 );
1930
1931 VOID
1932 NTAPI
1933 MiDecrementShareCount(
1934     IN PMMPFN Pfn1,
1935     IN PFN_NUMBER PageFrameIndex
1936 );
1937
1938 PFN_NUMBER
1939 NTAPI
1940 MiRemoveAnyPage(
1941     IN ULONG Color
1942 );
1943
1944 PFN_NUMBER
1945 NTAPI
1946 MiRemoveZeroPage(
1947     IN ULONG Color
1948 );
1949
1950 VOID
1951 NTAPI
1952 MiZeroPhysicalPage(
1953     IN PFN_NUMBER PageFrameIndex
1954 );
1955
1956 VOID
1957 NTAPI
1958 MiInsertPageInFreeList(
1959     IN PFN_NUMBER PageFrameIndex
1960 );
1961
1962 PFN_COUNT
1963 NTAPI
1964 MiDeleteSystemPageableVm(
1965     IN PMMPTE PointerPte,
1966     IN PFN_NUMBER PageCount,
1967     IN ULONG Flags,
1968     OUT PPFN_NUMBER ValidPages
1969 );
1970
1971 ULONG
1972 NTAPI
1973 MiGetPageProtection(
1974     IN PMMPTE PointerPte
1975 );
1976
1977 PLDR_DATA_TABLE_ENTRY
1978 NTAPI
1979 MiLookupDataTableEntry(
1980     IN PVOID Address
1981 );
1982
1983 VOID
1984 NTAPI
1985 MiInitializeDriverLargePageList(
1986     VOID
1987 );
1988
1989 VOID
1990 NTAPI
1991 MiInitializeLargePageSupport(
1992     VOID
1993 );
1994
1995 VOID
1996 NTAPI
1997 MiSyncCachedRanges(
1998     VOID
1999 );
2000
2001 BOOLEAN
2002 NTAPI
2003 MiIsPfnInUse(
2004     IN PMMPFN Pfn1
2005 );
2006
2007 PMMVAD
2008 NTAPI
2009 MiLocateAddress(
2010     IN PVOID VirtualAddress
2011 );
2012
2013 TABLE_SEARCH_RESULT
2014 NTAPI
2015 MiCheckForConflictingNode(
2016     IN ULONG_PTR StartVpn,
2017     IN ULONG_PTR EndVpn,
2018     IN PMM_AVL_TABLE Table,
2019     OUT PMMADDRESS_NODE *NodeOrParent
2020 );
2021
2022 TABLE_SEARCH_RESULT
2023 NTAPI
2024 MiFindEmptyAddressRangeDownTree(
2025     IN SIZE_T Length,
2026     IN ULONG_PTR BoundaryAddress,
2027     IN ULONG_PTR Alignment,
2028     IN PMM_AVL_TABLE Table,
2029     OUT PULONG_PTR Base,
2030     OUT PMMADDRESS_NODE *Parent
2031 );
2032
2033 NTSTATUS
2034 NTAPI
2035 MiFindEmptyAddressRangeDownBasedTree(
2036     IN SIZE_T Length,
2037     IN ULONG_PTR BoundaryAddress,
2038     IN ULONG_PTR Alignment,
2039     IN PMM_AVL_TABLE Table,
2040     OUT PULONG_PTR Base
2041 );
2042
2043 TABLE_SEARCH_RESULT
2044 NTAPI
2045 MiFindEmptyAddressRangeInTree(
2046     IN SIZE_T Length,
2047     IN ULONG_PTR Alignment,
2048     IN PMM_AVL_TABLE Table,
2049     OUT PMMADDRESS_NODE *PreviousVad,
2050     OUT PULONG_PTR Base
2051 );
2052
2053 NTSTATUS
2054 NTAPI
2055 MiCheckSecuredVad(
2056     IN PMMVAD Vad,
2057     IN PVOID Base,
2058     IN SIZE_T Size,
2059     IN ULONG ProtectionMask
2060 );
2061
2062 VOID
2063 NTAPI
2064 MiInsertVad(
2065     _Inout_ PMMVAD Vad,
2066     _Inout_ PMM_AVL_TABLE VadRoot);
2067
2068 NTSTATUS
2069 NTAPI
2070 MiInsertVadEx(
2071     _Inout_ PMMVAD Vad,
2072     _In_ ULONG_PTR *BaseAddress,
2073     _In_ SIZE_T ViewSize,
2074     _In_ ULONG_PTR HighestAddress,
2075     _In_ ULONG_PTR Alignment,
2076     _In_ ULONG AllocationType);
2077
2078 VOID
2079 NTAPI
2080 MiInsertBasedSection(
2081     IN PSECTION Section
2082 );
2083
2084 NTSTATUS
2085 NTAPI
2086 MiUnmapViewOfSection(
2087     IN PEPROCESS Process,
2088     IN PVOID BaseAddress,
2089     IN ULONG Flags
2090 );
2091
2092 NTSTATUS
2093 NTAPI
2094 MiRosUnmapViewOfSection(
2095     IN PEPROCESS Process,
2096     IN PVOID BaseAddress,
2097     IN BOOLEAN SkipDebuggerNotify
2098 );
2099
2100 VOID
2101 NTAPI
2102 MiInsertNode(
2103     IN PMM_AVL_TABLE Table,
2104     IN PMMADDRESS_NODE NewNode,
2105     PMMADDRESS_NODE Parent,
2106     TABLE_SEARCH_RESULT Result
2107 );
2108
2109 VOID
2110 NTAPI
2111 MiRemoveNode(
2112     IN PMMADDRESS_NODE Node,
2113     IN PMM_AVL_TABLE Table
2114 );
2115
2116 PMMADDRESS_NODE
2117 NTAPI
2118 MiGetPreviousNode(
2119     IN PMMADDRESS_NODE Node
2120 );
2121
2122 PMMADDRESS_NODE
2123 NTAPI
2124 MiGetNextNode(
2125     IN PMMADDRESS_NODE Node
2126 );
2127
2128 BOOLEAN
2129 NTAPI
2130 MiInitializeSystemSpaceMap(
2131     IN PMMSESSION InputSession OPTIONAL
2132 );
2133
2134 VOID
2135 NTAPI
2136 MiSessionRemoveProcess(
2137     VOID
2138 );
2139
2140 VOID
2141 NTAPI
2142 MiReleaseProcessReferenceToSessionDataPage(
2143     IN PMM_SESSION_SPACE SessionGlobal
2144 );
2145
2146 VOID
2147 NTAPI
2148 MiSessionAddProcess(
2149     IN PEPROCESS NewProcess
2150 );
2151
2152 NTSTATUS
2153 NTAPI
2154 MiSessionCommitPageTables(
2155     IN PVOID StartVa,
2156     IN PVOID EndVa
2157 );
2158
2159 ULONG
2160 NTAPI
2161 MiMakeProtectionMask(
2162     IN ULONG Protect
2163 );
2164
2165 VOID
2166 NTAPI
2167 MiDeleteVirtualAddresses(
2168     IN ULONG_PTR Va,
2169     IN ULONG_PTR EndingAddress,
2170     IN PMMVAD Vad
2171 );
2172
2173 VOID
2174 NTAPI
2175 MiDeletePte(
2176     IN PMMPTE PointerPte,
2177     IN PVOID VirtualAddress,
2178     IN PEPROCESS CurrentProcess,
2179     IN PMMPTE PrototypePte
2180 );
2181
2182 ULONG
2183 NTAPI
2184 MiMakeSystemAddressValid(
2185     IN PVOID PageTableVirtualAddress,
2186     IN PEPROCESS CurrentProcess
2187 );
2188
2189 ULONG
2190 NTAPI
2191 MiMakeSystemAddressValidPfn(
2192     IN PVOID VirtualAddress,
2193     IN KIRQL OldIrql
2194 );
2195
2196 VOID
2197 NTAPI
2198 MiRemoveMappedView(
2199     IN PEPROCESS CurrentProcess,
2200     IN PMMVAD Vad
2201 );
2202
2203 PSUBSECTION
2204 NTAPI
2205 MiLocateSubsection(
2206     IN PMMVAD Vad,
2207     IN ULONG_PTR Vpn
2208 );
2209
2210 VOID
2211 NTAPI
2212 MiDeleteARM3Section(
2213     PVOID ObjectBody
2214 );
2215
2216 NTSTATUS
2217 NTAPI
2218 MiQueryMemorySectionName(
2219     IN HANDLE ProcessHandle,
2220     IN PVOID BaseAddress,
2221     OUT PVOID MemoryInformation,
2222     IN SIZE_T MemoryInformationLength,
2223     OUT PSIZE_T ReturnLength
2224 );
2225
2226 NTSTATUS
2227 NTAPI
2228 MiRosUnmapViewInSystemSpace(
2229     IN PVOID MappedBase
2230 );
2231
2232 POOL_TYPE
2233 NTAPI
2234 MmDeterminePoolType(
2235     IN PVOID PoolAddress
2236 );
2237
2238 VOID
2239 NTAPI
2240 MiMakePdeExistAndMakeValid(
2241     IN PMMPDE PointerPde,
2242     IN PEPROCESS TargetProcess,
2243     IN KIRQL OldIrql
2244 );
2245
2246 //
2247 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2248 // free pages are available. In some scenarios, we don't/can't run that piece of
2249 // code and would rather only have a real zero page. If we can't have a zero page,
2250 // then we'd like to have our own code to grab a free page and zero it out, by
2251 // using MiRemoveAnyPage. This macro implements this.
2252 //
2253 FORCEINLINE
2254 PFN_NUMBER
2255 MiRemoveZeroPageSafe(IN ULONG Color)
2256 {
2257     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2258     return 0;
2259 }
2260
2261 /* EOF */