ntoskrnl/mm/ARM3/miarm.h

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
   5  * PURPOSE:         ARM Memory Manager Header
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 #pragma once
  10
  11 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
  12
  13 /* Make the code cleaner with some definitions for size multiples */
  14 #define _1KB (1024u)
  15 #define _1MB (1024 * _1KB)
  16 #define _1GB (1024 * _1MB)
  17
  18 /* Everyone loves 64K */
  19 #define _64K (64 * _1KB)
  20
  21 /* Area mapped by a PDE */
  22 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
  23
  24 /* Size of a page table */
  25 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
  26
  27 /* Size of a page directory */
  28 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
  29
  30 /* Stop using these! */
  31 #define PD_COUNT  PPE_PER_PAGE
  32 #define PDE_COUNT PDE_PER_PAGE
  33 #define PTE_COUNT PTE_PER_PAGE
  34
  35 /* Size of all page directories for a process */
  36 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
  37 #ifdef _M_IX86
  38 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
  39 #endif
  40
  41 //
  42 // Protection Bits part of the internal memory manager Protection Mask, from:
  43 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
  44 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
  45 // and public assertions.
  46 //
  47 #define MM_ZERO_ACCESS         0
  48 #define MM_READONLY            1
  49 #define MM_EXECUTE             2
  50 #define MM_EXECUTE_READ        3
  51 #define MM_READWRITE           4
  52 #define MM_WRITECOPY           5
  53 #define MM_EXECUTE_READWRITE   6
  54 #define MM_EXECUTE_WRITECOPY   7
  55 #define MM_PROTECT_ACCESS      7
  56
  57 //
  58 // These are flags on top of the actual protection mask
  59 //
  60 #define MM_NOCACHE            0x08
  61 #define MM_GUARDPAGE          0x10
  62 #define MM_WRITECOMBINE       0x18
  63 #define MM_PROTECT_SPECIAL    0x18
  64
  65 //
  66 // These are special cases
  67 //
  68 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
  69 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
  70 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
  71 #define MM_INVALID_PROTECTION  0xFFFFFFFF
  72
  73 //
  74 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
  75 // The Memory Manager's definition define the attributes that must be preserved
  76 // and these PTE definitions describe the attributes in the hardware sense. This
  77 // helps deal with hardware differences between the actual boolean expression of
  78 // the argument.
  79 //
  80 // For example, in the logical attributes, we want to express read-only as a flag
  81 // but on x86, it is writability that must be set. On the other hand, on x86, just
  82 // like in the kernel, it is disabling the caches that requires a special flag,
  83 // while on certain architectures such as ARM, it is enabling the cache which
  84 // requires a flag.
  85 //
  86 #if defined(_M_IX86) || defined(_M_AMD64)
  87 //
  88 // Access Flags
  89 //
  90 #define PTE_READONLY            0 // Doesn't exist on x86
  91 #define PTE_EXECUTE             0 // Not worrying about NX yet
  92 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
  93 #define PTE_READWRITE           0x2
  94 #define PTE_WRITECOPY           0x200
  95 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
  96 #define PTE_EXECUTE_WRITECOPY   0x200
  97 #define PTE_PROTOTYPE           0x400
  98
  99 //
 100 // State Flags
 101 //
 102 #define PTE_VALID               0x1
 103 #define PTE_ACCESSED            0x20
 104 #define PTE_DIRTY               0x40
 105
 106 //
 107 // Cache flags
 108 //
 109 #define PTE_ENABLE_CACHE        0
 110 #define PTE_DISABLE_CACHE       0x10
 111 #define PTE_WRITECOMBINED_CACHE 0x10
 112 #elif defined(_M_ARM)
 113 #define PTE_READONLY            0x200
 114 #define PTE_EXECUTE             0 // Not worrying about NX yet
 115 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 116 #define PTE_READWRITE           0 // Doesn't exist on ARM
 117 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
 118 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
 119 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
 120 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
 121 //
 122 // Cache flags
 123 //
 124 #define PTE_ENABLE_CACHE        0
 125 #define PTE_DISABLE_CACHE       0x10
 126 #define PTE_WRITECOMBINED_CACHE 0x10
 127 #else
 128 #error Define these please!
 129 #endif
 130
 131 extern const ULONG_PTR MmProtectToPteMask[32];
 132 extern const ULONG MmProtectToValue[32];
 133
 134 //
 135 // Assertions for session images, addresses, and PTEs
 136 //
 137 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
 138     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
 139
 140 #define MI_IS_SESSION_ADDRESS(Address) \
 141     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
 142
 143 #define MI_IS_SESSION_PTE(Pte) \
 144     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
 145
 146 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
 147     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
 148
 149 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
 150     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
 151
 152 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
 153     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
 154
 155 //
 156 // Creates a software PTE with the given protection
 157 //
 158 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
 159
 160 //
 161 // Marks a PTE as deleted
 162 //
 163 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
 164 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
 165
 166 //
 167 // Special values for LoadedImports
 168 //
 169 #ifdef _WIN64
 170 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
 171 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
 172 #else
 173 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
 174 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
 175 #endif
 176 #define MM_SYSLDR_SINGLE_ENTRY 0x1
 177
 178 //
 179 // Number of initial session IDs
 180 //
 181 #define MI_INITIAL_SESSION_IDS  64
 182
 183 #if defined(_M_IX86) || defined(_M_ARM)
 184 //
 185 // PFN List Sentinel
 186 //
 187 #define LIST_HEAD 0xFFFFFFFF
 188
 189 //
 190 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
 191 // we need a manual definition suited to the number of bits in the PteFrame.
 192 // This is used as a LIST_HEAD for the colored list
 193 //
 194 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
 195 #elif defined(_M_AMD64)
 196 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
 197 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
 198 #else
 199 #error Define these please!
 200 #endif
 201
 202 //
 203 // Special IRQL value (found in assertions)
 204 //
 205 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 206
 207 //
 208 // Returns the color of a page
 209 //
 210 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
 211 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
 212 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
 213
 214 //
 215 // Prototype PTEs that don't yet have a pagefile association
 216 //
 217 #ifdef _WIN64
 218 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
 219 #else
 220 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
 221 #endif
 222
 223 //
 224 // Number of session data and tag pages
 225 //
 226 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 227 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 228
 229 //
 230 // Used by MiCheckSecuredVad
 231 //
 232 #define MM_READ_WRITE_ALLOWED   11
 233 #define MM_READ_ONLY_ALLOWED    10
 234 #define MM_NO_ACCESS_ALLOWED    01
 235 #define MM_DELETE_CHECK         85
 236
 237 //
 238 // System views are binned into 64K chunks
 239 //
 240 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
 241
 242 //
 243 // FIXFIX: These should go in ex.h after the pool merge
 244 //
 245 #ifdef _WIN64
 246 #define POOL_BLOCK_SIZE 16
 247 #else
 248 #define POOL_BLOCK_SIZE  8
 249 #endif
 250 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
 251 #define BASE_POOL_TYPE_MASK 1
 252 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
 253
 254 //
 255 // Pool debugging/analysis/tracing flags
 256 //
 257 #define POOL_FLAG_CHECK_TIMERS 0x1
 258 #define POOL_FLAG_CHECK_WORKERS 0x2
 259 #define POOL_FLAG_CHECK_RESOURCES 0x4
 260 #define POOL_FLAG_VERIFIER 0x8
 261 #define POOL_FLAG_CHECK_DEADLOCK 0x10
 262 #define POOL_FLAG_SPECIAL_POOL 0x20
 263 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
 264 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
 265
 266 //
 267 // BAD_POOL_HEADER codes during pool bugcheck
 268 //
 269 #define POOL_CORRUPTED_LIST 3
 270 #define POOL_SIZE_OR_INDEX_MISMATCH 5
 271 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
 272 #define POOL_HEADER_NOT_ALIGNED 7
 273 #define POOL_HEADER_IS_ZERO 8
 274 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
 275 #define POOL_ENTRY_NOT_FOUND 10
 276
 277 //
 278 // BAD_POOL_CALLER codes during pool bugcheck
 279 //
 280 #define POOL_ENTRY_CORRUPTED 1
 281 #define POOL_ENTRY_ALREADY_FREE 6
 282 #define POOL_ENTRY_NOT_ALLOCATED 7
 283 #define POOL_ALLOC_IRQL_INVALID 8
 284 #define POOL_FREE_IRQL_INVALID 9
 285 #define POOL_BILLED_PROCESS_INVALID 13
 286 #define POOL_HEADER_SIZE_INVALID 32
 287
 288 typedef struct _POOL_DESCRIPTOR
 289 {
 290     POOL_TYPE PoolType;
 291     ULONG PoolIndex;
 292     ULONG RunningAllocs;
 293     ULONG RunningDeAllocs;
 294     ULONG TotalPages;
 295     ULONG TotalBigPages;
 296     ULONG Threshold;
 297     PVOID LockAddress;
 298     PVOID PendingFrees;
 299     LONG PendingFreeDepth;
 300     SIZE_T TotalBytes;
 301     SIZE_T Spare0;
 302     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
 303 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 304
 305 typedef struct _POOL_HEADER
 306 {
 307     union
 308     {
 309         struct
 310         {
 311 #ifdef _WIN64
 312             USHORT PreviousSize:8;
 313             USHORT PoolIndex:8;
 314             USHORT BlockSize:8;
 315             USHORT PoolType:8;
 316 #else
 317             USHORT PreviousSize:9;
 318             USHORT PoolIndex:7;
 319             USHORT BlockSize:9;
 320             USHORT PoolType:7;
 321 #endif
 322         };
 323         ULONG Ulong1;
 324     };
 325 #ifdef _WIN64
 326     ULONG PoolTag;
 327 #endif
 328     union
 329     {
 330 #ifdef _WIN64
 331         PEPROCESS ProcessBilled;
 332 #else
 333         ULONG PoolTag;
 334 #endif
 335         struct
 336         {
 337             USHORT AllocatorBackTraceIndex;
 338             USHORT PoolTagHash;
 339         };
 340     };
 341 } POOL_HEADER, *PPOOL_HEADER;
 342
 343 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
 344 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
 345
 346 typedef struct _POOL_TRACKER_TABLE
 347 {
 348     ULONG Key;
 349     LONG NonPagedAllocs;
 350     LONG NonPagedFrees;
 351     SIZE_T NonPagedBytes;
 352     LONG PagedAllocs;
 353     LONG PagedFrees;
 354     SIZE_T PagedBytes;
 355 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
 356
 357 typedef struct _POOL_TRACKER_BIG_PAGES
 358 {
 359     PVOID Va;
 360     ULONG Key;
 361     ULONG NumberOfPages;
 362     PVOID QuotaObject;
 363 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
 364
 365 extern ULONG ExpNumberOfPagedPools;
 366 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
 367 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
 368 extern PPOOL_TRACKER_TABLE PoolTrackTable;
 369
 370 //
 371 // END FIXFIX
 372 //
 373
 374 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
 375 {
 376     LIST_ENTRY Links;
 377     UNICODE_STRING BaseName;
 378 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
 379
 380 typedef enum _MMSYSTEM_PTE_POOL_TYPE
 381 {
 382     SystemPteSpace,
 383     NonPagedPoolExpansion,
 384     MaximumPtePoolTypes
 385 } MMSYSTEM_PTE_POOL_TYPE;
 386
 387 typedef enum _MI_PFN_CACHE_ATTRIBUTE
 388 {
 389     MiNonCached,
 390     MiCached,
 391     MiWriteCombined,
 392     MiNotMapped
 393 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
 394
 395 typedef struct _PHYSICAL_MEMORY_RUN
 396 {
 397     PFN_NUMBER BasePage;
 398     PFN_NUMBER PageCount;
 399 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
 400
 401 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
 402 {
 403     ULONG NumberOfRuns;
 404     PFN_NUMBER NumberOfPages;
 405     PHYSICAL_MEMORY_RUN Run[1];
 406 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
 407
 408 typedef struct _MMCOLOR_TABLES
 409 {
 410     PFN_NUMBER Flink;
 411     PVOID Blink;
 412     PFN_NUMBER Count;
 413 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 414
 415 typedef struct _MI_LARGE_PAGE_RANGES
 416 {
 417     PFN_NUMBER StartFrame;
 418     PFN_NUMBER LastFrame;
 419 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
 420
 421 typedef struct _MMVIEW
 422 {
 423     ULONG_PTR Entry;
 424     PCONTROL_AREA ControlArea;
 425 } MMVIEW, *PMMVIEW;
 426
 427 typedef struct _MMSESSION
 428 {
 429     KGUARDED_MUTEX SystemSpaceViewLock;
 430     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
 431     PCHAR SystemSpaceViewStart;
 432     PMMVIEW SystemSpaceViewTable;
 433     ULONG SystemSpaceHashSize;
 434     ULONG SystemSpaceHashEntries;
 435     ULONG SystemSpaceHashKey;
 436     ULONG BitmapFailures;
 437     PRTL_BITMAP SystemSpaceBitMap;
 438 } MMSESSION, *PMMSESSION;
 439
 440 typedef struct _MM_SESSION_SPACE_FLAGS
 441 {
 442     ULONG Initialized:1;
 443     ULONG DeletePending:1;
 444     ULONG Filler:30;
 445 } MM_SESSION_SPACE_FLAGS;
 446
 447 typedef struct _MM_SESSION_SPACE
 448 {
 449     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
 450     LONG ReferenceCount;
 451     union
 452     {
 453         ULONG LongFlags;
 454         MM_SESSION_SPACE_FLAGS Flags;
 455     } u;
 456     ULONG SessionId;
 457     LIST_ENTRY ProcessList;
 458     LARGE_INTEGER LastProcessSwappedOutTime;
 459     PFN_NUMBER SessionPageDirectoryIndex;
 460     SIZE_T NonPageablePages;
 461     SIZE_T CommittedPages;
 462     PVOID PagedPoolStart;
 463     PVOID PagedPoolEnd;
 464     PMMPDE PagedPoolBasePde;
 465     ULONG Color;
 466     LONG ResidentProcessCount;
 467     ULONG SessionPoolAllocationFailures[4];
 468     LIST_ENTRY ImageList;
 469     LCID LocaleId;
 470     ULONG AttachCount;
 471     KEVENT AttachEvent;
 472     PEPROCESS LastProcess;
 473     LONG ProcessReferenceToSession;
 474     LIST_ENTRY WsListEntry;
 475     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
 476     MMSESSION Session;
 477     KGUARDED_MUTEX PagedPoolMutex;
 478     MM_PAGED_POOL_INFO PagedPoolInfo;
 479     MMSUPPORT Vm;
 480     PMMWSLE Wsle;
 481     PDRIVER_UNLOAD Win32KDriverUnload;
 482     POOL_DESCRIPTOR PagedPool;
 483 #if defined (_M_AMD64)
 484     MMPDE PageDirectory;
 485 #else
 486     PMMPDE PageTables;
 487 #endif
 488 #if defined (_M_AMD64)
 489     PMMPTE SpecialPoolFirstPte;
 490     PMMPTE SpecialPoolLastPte;
 491     PMMPTE NextPdeForSpecialPoolExpansion;
 492     PMMPTE LastPdeForSpecialPoolExpansion;
 493     PFN_NUMBER SpecialPagesInUse;
 494 #endif
 495     LONG ImageLoadingCount;
 496 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
 497
 498 extern PMM_SESSION_SPACE MmSessionSpace;
 499 extern MMPTE HyperTemplatePte;
 500 extern MMPDE ValidKernelPde;
 501 extern MMPTE ValidKernelPte;
 502 extern MMPDE ValidKernelPdeLocal;
 503 extern MMPTE ValidKernelPteLocal;
 504 extern MMPDE DemandZeroPde;
 505 extern MMPTE DemandZeroPte;
 506 extern MMPTE PrototypePte;
 507 extern MMPTE MmDecommittedPte;
 508 extern BOOLEAN MmLargeSystemCache;
 509 extern BOOLEAN MmZeroPageFile;
 510 extern BOOLEAN MmProtectFreedNonPagedPool;
 511 extern BOOLEAN MmTrackLockedPages;
 512 extern BOOLEAN MmTrackPtes;
 513 extern BOOLEAN MmDynamicPfn;
 514 extern BOOLEAN MmMirroring;
 515 extern BOOLEAN MmMakeLowMemory;
 516 extern BOOLEAN MmEnforceWriteProtection;
 517 extern SIZE_T MmAllocationFragment;
 518 extern ULONG MmConsumedPoolPercentage;
 519 extern ULONG MmVerifyDriverBufferType;
 520 extern ULONG MmVerifyDriverLevel;
 521 extern WCHAR MmVerifyDriverBuffer[512];
 522 extern WCHAR MmLargePageDriverBuffer[512];
 523 extern LIST_ENTRY MiLargePageDriverList;
 524 extern BOOLEAN MiLargePageAllDrivers;
 525 extern ULONG MmVerifyDriverBufferLength;
 526 extern ULONG MmLargePageDriverBufferLength;
 527 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
 528 extern SIZE_T MmMaximumNonPagedPoolInBytes;
 529 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
 530 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
 531 extern PVOID MmNonPagedSystemStart;
 532 extern PVOID MmNonPagedPoolStart;
 533 extern PVOID MmNonPagedPoolExpansionStart;
 534 extern PVOID MmNonPagedPoolEnd;
 535 extern SIZE_T MmSizeOfPagedPoolInBytes;
 536 extern PVOID MmPagedPoolStart;
 537 extern PVOID MmPagedPoolEnd;
 538 extern PVOID MmSessionBase;
 539 extern SIZE_T MmSessionSize;
 540 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
 541 extern PMMPTE MiFirstReservedZeroingPte;
 542 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
 543 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
 544 extern SIZE_T MmBootImageSize;
 545 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
 546 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
 547 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
 548 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
 549 extern ULONG_PTR MxPfnAllocation;
 550 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
 551 extern RTL_BITMAP MiPfnBitMap;
 552 extern KGUARDED_MUTEX MmPagedPoolMutex;
 553 extern KGUARDED_MUTEX MmSectionCommitMutex;
 554 extern PVOID MmPagedPoolStart;
 555 extern PVOID MmPagedPoolEnd;
 556 extern PVOID MmNonPagedSystemStart;
 557 extern PVOID MiSystemViewStart;
 558 extern SIZE_T MmSystemViewSize;
 559 extern PVOID MmSessionBase;
 560 extern PVOID MiSessionSpaceEnd;
 561 extern PMMPTE MiSessionImagePteStart;
 562 extern PMMPTE MiSessionImagePteEnd;
 563 extern PMMPTE MiSessionBasePte;
 564 extern PMMPTE MiSessionLastPte;
 565 extern SIZE_T MmSizeOfPagedPoolInBytes;
 566 extern PMMPDE MmSystemPagePtes;
 567 extern PVOID MmSystemCacheStart;
 568 extern PVOID MmSystemCacheEnd;
 569 extern MMSUPPORT MmSystemCacheWs;
 570 extern SIZE_T MmAllocatedNonPagedPool;
 571 extern ULONG MmSpecialPoolTag;
 572 extern PVOID MmHyperSpaceEnd;
 573 extern PMMWSL MmSystemCacheWorkingSetList;
 574 extern SIZE_T MmMinimumNonPagedPoolSize;
 575 extern ULONG MmMinAdditionNonPagedPoolPerMb;
 576 extern SIZE_T MmDefaultMaximumNonPagedPool;
 577 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
 578 extern ULONG MmSecondaryColors;
 579 extern ULONG MmSecondaryColorMask;
 580 extern ULONG MmNumberOfSystemPtes;
 581 extern ULONG MmMaximumNonPagedPoolPercent;
 582 extern ULONG MmLargeStackSize;
 583 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
 584 extern MMPFNLIST MmStandbyPageListByPriority[8];
 585 extern ULONG MmProductType;
 586 extern MM_SYSTEMSIZE MmSystemSize;
 587 extern PKEVENT MiLowMemoryEvent;
 588 extern PKEVENT MiHighMemoryEvent;
 589 extern PKEVENT MiLowPagedPoolEvent;
 590 extern PKEVENT MiHighPagedPoolEvent;
 591 extern PKEVENT MiLowNonPagedPoolEvent;
 592 extern PKEVENT MiHighNonPagedPoolEvent;
 593 extern PFN_NUMBER MmLowMemoryThreshold;
 594 extern PFN_NUMBER MmHighMemoryThreshold;
 595 extern PFN_NUMBER MiLowPagedPoolThreshold;
 596 extern PFN_NUMBER MiHighPagedPoolThreshold;
 597 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
 598 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 599 extern PFN_NUMBER MmMinimumFreePages;
 600 extern PFN_NUMBER MmPlentyFreePages;
 601 extern SIZE_T MmMinimumStackCommitInBytes;
 602 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 603 extern PFN_NUMBER MmResidentAvailablePages;
 604 extern PFN_NUMBER MmResidentAvailableAtInit;
 605 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
 606 extern PFN_NUMBER MmTotalSystemDriverPages;
 607 extern ULONG MmCritsectTimeoutSeconds;
 608 extern PVOID MiSessionImageStart;
 609 extern PVOID MiSessionImageEnd;
 610 extern PMMPTE MiHighestUserPte;
 611 extern PMMPDE MiHighestUserPde;
 612 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
 613 extern PMMPTE MmSharedUserDataPte;
 614 extern LIST_ENTRY MmProcessList;
 615 extern BOOLEAN MmZeroingPageThreadActive;
 616 extern KEVENT MmZeroingPageEvent;
 617 extern ULONG MmSystemPageColor;
 618 extern ULONG MmProcessColorSeed;
 619 extern PMMWSL MmWorkingSetList;
 620 extern PFN_NUMBER MiNumberOfFreePages;
 621 extern SIZE_T MmSessionViewSize;
 622 extern SIZE_T MmSessionPoolSize;
 623 extern SIZE_T MmSessionImageSize;
 624 extern PVOID MiSystemViewStart;
 625 extern PVOID MiSessionPoolEnd;     // 0xBE000000
 626 extern PVOID MiSessionPoolStart;   // 0xBD000000
 627 extern PVOID MiSessionViewStart;   // 0xBE000000
 628 extern PVOID MiSessionSpaceWs;
 629 extern ULONG MmMaximumDeadKernelStacks;
 630 extern SLIST_HEADER MmDeadStackSListHead;
 631 extern MM_AVL_TABLE MmSectionBasedRoot;
 632 extern KGUARDED_MUTEX MmSectionBasedMutex;
 633 extern PVOID MmHighSectionBase;
 634 extern SIZE_T MmSystemLockPagesCount;
 635 extern ULONG_PTR MmSubsectionBase;
 636 extern LARGE_INTEGER MmCriticalSectionTimeout;
 637 extern LIST_ENTRY MmWorkingSetExpansionHead;
 638 extern KSPIN_LOCK MmExpansionLock;
 639 extern PETHREAD MiExpansionLockOwner;
 640
 641 FORCEINLINE
 642 BOOLEAN
 643 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
 644 {
 645     return ((MemoryType == LoaderFree) ||
 646             (MemoryType == LoaderLoadedProgram) ||
 647             (MemoryType == LoaderFirmwareTemporary) ||
 648             (MemoryType == LoaderOsloaderStack));
 649 }
 650
 651 FORCEINLINE
 652 BOOLEAN
 653 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
 654 {
 655     return ((MemoryType == LoaderFirmwarePermanent) ||
 656             (MemoryType == LoaderSpecialMemory) ||
 657             (MemoryType == LoaderHALCachedMemory) ||
 658             (MemoryType == LoaderBBTMemory));
 659 }
 660
 661 #ifdef _M_AMD64
 662 FORCEINLINE
 663 BOOLEAN
 664 MiIsUserPxe(PVOID Address)
 665 {
 666     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
 667 }
 668
 669 FORCEINLINE
 670 BOOLEAN
 671 MiIsUserPpe(PVOID Address)
 672 {
 673     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
 674 }
 675
 676 FORCEINLINE
 677 BOOLEAN
 678 MiIsUserPde(PVOID Address)
 679 {
 680     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
 681 }
 682
 683 FORCEINLINE
 684 BOOLEAN
 685 MiIsUserPte(PVOID Address)
 686 {
 687     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
 688 }
 689 #else
 690 FORCEINLINE
 691 BOOLEAN
 692 MiIsUserPde(PVOID Address)
 693 {
 694     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
 695             (Address <= (PVOID)MiHighestUserPde));
 696 }
 697
 698 FORCEINLINE
 699 BOOLEAN
 700 MiIsUserPte(PVOID Address)
 701 {
 702     return (Address <= (PVOID)MiHighestUserPte);
 703 }
 704 #endif
 705
 706 //
 707 // Figures out the hardware bits for a PTE
 708 //
 709 FORCEINLINE
 710 ULONG_PTR
 711 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
 712 {
 713     MMPTE TempPte;
 714
 715     /* Start fresh */
 716     TempPte.u.Long = 0;
 717
 718     /* Make it valid and accessed */
 719     TempPte.u.Hard.Valid = TRUE;
 720     MI_MAKE_ACCESSED_PAGE(&TempPte);
 721
 722     /* Is this for user-mode? */
 723     if (
 724 #if (_MI_PAGING_LEVELS == 4)
 725         MiIsUserPxe(PointerPte) ||
 726 #endif
 727 #if (_MI_PAGING_LEVELS >= 3)
 728         MiIsUserPpe(PointerPte) ||
 729 #endif
 730         MiIsUserPde(PointerPte) ||
 731         MiIsUserPte(PointerPte))
 732     {
 733         /* Set the owner bit */
 734         MI_MAKE_OWNER_PAGE(&TempPte);
 735     }
 736
 737     /* FIXME: We should also set the global bit */
 738
 739     /* Return the protection */
 740     return TempPte.u.Long;
 741 }
 742
 743 //
 744 // Creates a valid kernel PTE with the given protection
 745 //
 746 FORCEINLINE
 747 VOID
 748 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
 749                             IN PMMPTE MappingPte,
 750                             IN ULONG_PTR ProtectionMask,
 751                             IN PFN_NUMBER PageFrameNumber)
 752 {
 753     /* Only valid for kernel, non-session PTEs */
 754     ASSERT(MappingPte > MiHighestUserPte);
 755     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
 756     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 757
 758     /* Start fresh */
 759     *NewPte = ValidKernelPte;
 760
 761     /* Set the protection and page */
 762     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 763     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 764 }
 765
 766 //
 767 // Creates a valid PTE with the given protection
 768 //
 769 FORCEINLINE
 770 VOID
 771 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
 772                      IN PMMPTE MappingPte,
 773                      IN ULONG_PTR ProtectionMask,
 774                      IN PFN_NUMBER PageFrameNumber)
 775 {
 776     /* Set the protection and page */
 777     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
 778     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 779     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 780 }
 781
 782 //
 783 // Creates a valid user PTE with the given protection
 784 //
 785 FORCEINLINE
 786 VOID
 787 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
 788                           IN PMMPTE MappingPte,
 789                           IN ULONG_PTR ProtectionMask,
 790                           IN PFN_NUMBER PageFrameNumber)
 791 {
 792     /* Only valid for kernel, non-session PTEs */
 793     ASSERT(MappingPte <= MiHighestUserPte);
 794
 795     /* Start fresh */
 796     NewPte->u.Long = 0;
 797
 798     /* Set the protection and page */
 799     NewPte->u.Hard.Valid = TRUE;
 800     NewPte->u.Hard.Owner = TRUE;
 801     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 802     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 803 }
 804
 805 #ifndef _M_AMD64
 806 //
 807 // Builds a Prototype PTE for the address of the PTE
 808 //
 809 FORCEINLINE
 810 VOID
 811 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
 812                       IN PMMPTE PointerPte)
 813 {
 814     ULONG_PTR Offset;
 815
 816     /* Mark this as a prototype */
 817     NewPte->u.Long = 0;
 818     NewPte->u.Proto.Prototype = 1;
 819
 820     /*
 821      * Prototype PTEs are only valid in paged pool by design, this little trick
 822      * lets us only use 30 bits for the adress of the PTE, as long as the area
 823      * stays 1024MB At most.
 824      */
 825     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 826
 827     /*
 828      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
 829      * and the other 21 bits go in the "high"
 830      */
 831     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
 832     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
 833 }
 834
 835 //
 836 // Builds a Subsection PTE for the address of the Segment
 837 //
 838 FORCEINLINE
 839 VOID
 840 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
 841                        IN PVOID Segment)
 842 {
 843     ULONG_PTR Offset;
 844
 845     /* Mark this as a prototype */
 846     NewPte->u.Long = 0;
 847     NewPte->u.Subsect.Prototype = 1;
 848
 849     /*
 850      * Segments are only valid either in nonpaged pool. We store the 20 bit
 851      * difference either from the top or bottom of nonpaged pool, giving a
 852      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
 853      * 256MB.
 854      */
 855     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
 856     {
 857         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
 858         NewPte->u.Subsect.WhichPool = PagedPool;
 859     }
 860     else
 861     {
 862         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
 863         NewPte->u.Subsect.WhichPool = NonPagedPool;
 864     }
 865
 866     /*
 867      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
 868      * and the other 20 bits go in the "high"
 869      */
 870     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
 871     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
 872 }
 873
 874 FORCEINLINE
 875 BOOLEAN
 876 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
 877 {
 878     /// \todo Make this reasonable code, this is UGLY!
 879     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
 880 }
 881
 882 #endif
 883
 884 FORCEINLINE
 885 VOID
 886 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
 887                        _In_ PFN_NUMBER Page,
 888                        _In_ ULONG Protection)
 889 {
 890     NewPte->u.Long = 0;
 891     NewPte->u.Trans.Transition = 1;
 892     NewPte->u.Trans.Protection = Protection;
 893     NewPte->u.Trans.PageFrameNumber = Page;
 894 }
 895
 896 //
 897 // Returns if the page is physically resident (ie: a large page)
 898 // FIXFIX: CISC/x86 only?
 899 //
 900 FORCEINLINE
 901 BOOLEAN
 902 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
 903 {
 904     PMMPDE PointerPde;
 905
 906     /* Large pages are never paged out, always physically resident */
 907     PointerPde = MiAddressToPde(Address);
 908     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
 909 }
 910
 911 //
 912 // Writes a valid PTE
 913 //
 914 FORCEINLINE
 915 VOID
 916 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
 917                    IN MMPTE TempPte)
 918 {
 919     /* Write the valid PTE */
 920     ASSERT(PointerPte->u.Hard.Valid == 0);
 921     ASSERT(TempPte.u.Hard.Valid == 1);
 922     *PointerPte = TempPte;
 923 }
 924
 925 //
 926 // Updates a valid PTE
 927 //
 928 FORCEINLINE
 929 VOID
 930 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
 931                    IN MMPTE TempPte)
 932 {
 933     /* Write the valid PTE */
 934     ASSERT(PointerPte->u.Hard.Valid == 1);
 935     ASSERT(TempPte.u.Hard.Valid == 1);
 936     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
 937     *PointerPte = TempPte;
 938 }
 939
 940 //
 941 // Writes an invalid PTE
 942 //
 943 FORCEINLINE
 944 VOID
 945 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
 946                      IN MMPTE InvalidPte)
 947 {
 948     /* Write the invalid PTE */
 949     ASSERT(InvalidPte.u.Hard.Valid == 0);
 950     ASSERT(InvalidPte.u.Long != 0);
 951     *PointerPte = InvalidPte;
 952 }
 953
 954 //
 955 // Erase the PTE completely
 956 //
 957 FORCEINLINE
 958 VOID
 959 MI_ERASE_PTE(IN PMMPTE PointerPte)
 960 {
 961     /* Zero out the PTE */
 962     ASSERT(PointerPte->u.Long != 0);
 963     PointerPte->u.Long = 0;
 964 }
 965
 966 //
 967 // Writes a valid PDE
 968 //
 969 FORCEINLINE
 970 VOID
 971 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
 972                    IN MMPDE TempPde)
 973 {
 974     /* Write the valid PDE */
 975     ASSERT(PointerPde->u.Hard.Valid == 0);
 976     ASSERT(TempPde.u.Hard.Valid == 1);
 977     *PointerPde = TempPde;
 978 }
 979
 980 //
 981 // Writes an invalid PDE
 982 //
 983 FORCEINLINE
 984 VOID
 985 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
 986                      IN MMPDE InvalidPde)
 987 {
 988     /* Write the invalid PDE */
 989     ASSERT(InvalidPde.u.Hard.Valid == 0);
 990     ASSERT(InvalidPde.u.Long != 0);
 991     *PointerPde = InvalidPde;
 992 }
 993
 994 //
 995 // Checks if the thread already owns a working set
 996 //
 997 FORCEINLINE
 998 BOOLEAN
 999 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1000 {
1001     /* If any of these are held, return TRUE */
1002     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1003             (Thread->OwnsProcessWorkingSetShared) ||
1004             (Thread->OwnsSystemWorkingSetExclusive) ||
1005             (Thread->OwnsSystemWorkingSetShared) ||
1006             (Thread->OwnsSessionWorkingSetExclusive) ||
1007             (Thread->OwnsSessionWorkingSetShared));
1008 }
1009
1010 //
1011 // Checks if the process owns the working set lock
1012 //
1013 FORCEINLINE
1014 BOOLEAN
1015 MI_WS_OWNER(IN PEPROCESS Process)
1016 {
1017     /* Check if this process is the owner, and that the thread owns the WS */
1018     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1019     {
1020         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1021     }
1022     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1023     {
1024         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1025     }
1026     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1027             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1028              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1029 }
1030
1031 //
1032 // New ARM3<->RosMM PAGE Architecture
1033 //
1034 FORCEINLINE
1035 BOOLEAN
1036 MiIsRosSectionObject(IN PVOID Section)
1037 {
1038     PROS_SECTION_OBJECT RosSection = Section;
1039     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1040     return FALSE;
1041 }
1042
1043 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1044
1045 VOID
1046 NTAPI
1047 MiDecrementReferenceCount(
1048     IN PMMPFN Pfn1,
1049     IN PFN_NUMBER PageFrameIndex
1050 );
1051
1052 FORCEINLINE
1053 BOOLEAN
1054 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1055 {
1056     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1057 }
1058
1059 //
1060 // Locks the working set for the given process
1061 //
1062 FORCEINLINE
1063 VOID
1064 MiLockProcessWorkingSet(IN PEPROCESS Process,
1065                         IN PETHREAD Thread)
1066 {
1067     /* Shouldn't already be owning the process working set */
1068     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1069     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1070
1071     /* Block APCs, make sure that still nothing is already held */
1072     KeEnterGuardedRegion();
1073     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1074
1075     /* Lock the working set */
1076     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1077
1078     /* Now claim that we own the lock */
1079     ASSERT(!MI_IS_WS_UNSAFE(Process));
1080     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1081     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1082 }
1083
1084 FORCEINLINE
1085 VOID
1086 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1087                               IN PETHREAD Thread)
1088 {
1089     /* Shouldn't already be owning the process working set */
1090     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1091     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1092
1093     /* Block APCs, make sure that still nothing is already held */
1094     KeEnterGuardedRegion();
1095     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1096
1097     /* Lock the working set */
1098     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1099
1100     /* Now claim that we own the lock */
1101     ASSERT(!MI_IS_WS_UNSAFE(Process));
1102     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1103     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1104     Thread->OwnsProcessWorkingSetShared = TRUE;
1105 }
1106
1107 FORCEINLINE
1108 VOID
1109 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1110                               IN PETHREAD Thread)
1111 {
1112     /* Shouldn't already be owning the process working set */
1113     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1114
1115     /* APCs must be blocked, make sure that still nothing is already held */
1116     ASSERT(KeAreAllApcsDisabled() == TRUE);
1117     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1118
1119     /* Lock the working set */
1120     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1121
1122     /* Now claim that we own the lock */
1123     ASSERT(!MI_IS_WS_UNSAFE(Process));
1124     Process->Vm.Flags.AcquiredUnsafe = 1;
1125     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1126     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1127 }
1128
1129 //
1130 // Unlocks the working set for the given process
1131 //
1132 FORCEINLINE
1133 VOID
1134 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1135                           IN PETHREAD Thread)
1136 {
1137     /* Make sure we are the owner of a safe acquisition */
1138     ASSERT(MI_WS_OWNER(Process));
1139     ASSERT(!MI_IS_WS_UNSAFE(Process));
1140
1141     /* The thread doesn't own it anymore */
1142     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1143     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1144
1145     /* Release the lock and re-enable APCs */
1146     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1147     KeLeaveGuardedRegion();
1148 }
1149
1150 //
1151 // Unlocks the working set for the given process
1152 //
1153 FORCEINLINE
1154 VOID
1155 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1156                                 IN PETHREAD Thread)
1157 {
1158     /* Make sure we are the owner of a safe acquisition (because shared) */
1159     ASSERT(MI_WS_OWNER(Process));
1160     ASSERT(!MI_IS_WS_UNSAFE(Process));
1161
1162     /* Ensure we are in a shared acquisition */
1163     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1164     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1165
1166     /* Don't claim the lock anylonger */
1167     Thread->OwnsProcessWorkingSetShared = FALSE;
1168
1169     /* Release the lock and re-enable APCs */
1170     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1171     KeLeaveGuardedRegion();
1172 }
1173
1174 //
1175 // Unlocks the working set for the given process
1176 //
1177 FORCEINLINE
1178 VOID
1179 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1180                                 IN PETHREAD Thread)
1181 {
1182     /* Make sure we are the owner of an unsafe acquisition */
1183     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1184     ASSERT(KeAreAllApcsDisabled() == TRUE);
1185     ASSERT(MI_WS_OWNER(Process));
1186     ASSERT(MI_IS_WS_UNSAFE(Process));
1187
1188     /* No longer unsafe */
1189     Process->Vm.Flags.AcquiredUnsafe = 0;
1190
1191     /* The thread doesn't own it anymore */
1192     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1193     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1194
1195     /* Release the lock but don't touch APC state */
1196     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1197     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1198 }
1199
1200 //
1201 // Locks the working set
1202 //
1203 FORCEINLINE
1204 VOID
1205 MiLockWorkingSet(IN PETHREAD Thread,
1206                  IN PMMSUPPORT WorkingSet)
1207 {
1208     /* Block APCs */
1209     KeEnterGuardedRegion();
1210
1211     /* Working set should be in global memory */
1212     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1213
1214     /* Thread shouldn't already be owning something */
1215     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1216
1217     /* Lock this working set */
1218     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1219
1220     /* Which working set is this? */
1221     if (WorkingSet == &MmSystemCacheWs)
1222     {
1223         /* Own the system working set */
1224         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1225                (Thread->OwnsSystemWorkingSetShared == FALSE));
1226         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1227     }
1228     else if (WorkingSet->Flags.SessionSpace)
1229     {
1230         /* Own the session working set */
1231         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1232                (Thread->OwnsSessionWorkingSetShared == FALSE));
1233         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1234     }
1235     else
1236     {
1237         /* Own the process working set */
1238         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1239                (Thread->OwnsProcessWorkingSetShared == FALSE));
1240         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1241     }
1242 }
1243
1244 //
1245 // Unlocks the working set
1246 //
1247 FORCEINLINE
1248 VOID
1249 MiUnlockWorkingSet(IN PETHREAD Thread,
1250                    IN PMMSUPPORT WorkingSet)
1251 {
1252     /* Working set should be in global memory */
1253     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1254
1255     /* Which working set is this? */
1256     if (WorkingSet == &MmSystemCacheWs)
1257     {
1258         /* Release the system working set */
1259         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1260                (Thread->OwnsSystemWorkingSetShared == TRUE));
1261         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1262     }
1263     else if (WorkingSet->Flags.SessionSpace)
1264     {
1265         /* Release the session working set */
1266         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1267                (Thread->OwnsSessionWorkingSetShared == TRUE));
1268         Thread->OwnsSessionWorkingSetExclusive = 0;
1269     }
1270     else
1271     {
1272         /* Release the process working set */
1273         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1274                (Thread->OwnsProcessWorkingSetShared));
1275         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1276     }
1277
1278     /* Release the working set lock */
1279     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1280
1281     /* Unblock APCs */
1282     KeLeaveGuardedRegion();
1283 }
1284
1285 FORCEINLINE
1286 VOID
1287 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1288                                   IN PETHREAD Thread,
1289                                   OUT PBOOLEAN Safe,
1290                                   OUT PBOOLEAN Shared)
1291 {
1292     ASSERT(MI_WS_OWNER(Process));
1293
1294     /* Check if the current owner is unsafe */
1295     if (MI_IS_WS_UNSAFE(Process))
1296     {
1297         /* Release unsafely */
1298         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1299         *Safe = FALSE;
1300         *Shared = FALSE;
1301     }
1302     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1303     {
1304         /* Owner is safe and exclusive, release normally */
1305         MiUnlockProcessWorkingSet(Process, Thread);
1306         *Safe = TRUE;
1307         *Shared = FALSE;
1308     }
1309     else
1310     {
1311         /* Owner is shared (implies safe), release normally */
1312         MiUnlockProcessWorkingSetShared(Process, Thread);
1313         *Safe = TRUE;
1314         *Shared = TRUE;
1315     }
1316 }
1317
1318 FORCEINLINE
1319 VOID
1320 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1321                                 IN PETHREAD Thread,
1322                                 IN BOOLEAN Safe,
1323                                 IN BOOLEAN Shared)
1324 {
1325     /* Check if this was a safe lock or not */
1326     if (Safe)
1327     {
1328         if (Shared)
1329         {
1330             /* Reacquire safely & shared */
1331             MiLockProcessWorkingSetShared(Process, Thread);
1332         }
1333         else
1334         {
1335             /* Reacquire safely */
1336             MiLockProcessWorkingSet(Process, Thread);
1337         }
1338     }
1339     else
1340     {
1341         /* Unsafe lock cannot be shared */
1342         ASSERT(Shared == FALSE);
1343         /* Reacquire unsafely */
1344         MiLockProcessWorkingSetUnsafe(Process, Thread);
1345     }
1346 }
1347
1348 FORCEINLINE
1349 KIRQL
1350 MiAcquireExpansionLock(VOID)
1351 {
1352     KIRQL OldIrql;
1353
1354     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1355     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1356     ASSERT(MiExpansionLockOwner == NULL);
1357     MiExpansionLockOwner = PsGetCurrentThread();
1358     return OldIrql;
1359 }
1360
1361 FORCEINLINE
1362 VOID
1363 MiReleaseExpansionLock(KIRQL OldIrql)
1364 {
1365     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1366     MiExpansionLockOwner = NULL;
1367     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1368     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1369 }
1370
1371 //
1372 // Returns the ProtoPTE inside a VAD for the given VPN
1373 //
1374 FORCEINLINE
1375 PMMPTE
1376 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1377                              IN ULONG_PTR Vpn)
1378 {
1379     PMMPTE ProtoPte;
1380
1381     /* Find the offset within the VAD's prototype PTEs */
1382     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1383     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1384     return ProtoPte;
1385 }
1386
1387 //
1388 // Returns the PFN Database entry for the given page number
1389 // Warning: This is not necessarily a valid PFN database entry!
1390 //
1391 FORCEINLINE
1392 PMMPFN
1393 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1394 {
1395     /* Get the entry */
1396     return &MmPfnDatabase[Pfn];
1397 };
1398
1399 //
1400 // Drops a locked page without dereferencing it
1401 //
1402 FORCEINLINE
1403 VOID
1404 MiDropLockCount(IN PMMPFN Pfn1)
1405 {
1406     /* This page shouldn't be locked, but it should be valid */
1407     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1408     ASSERT(Pfn1->u2.ShareCount == 0);
1409
1410     /* Is this the last reference to the page */
1411     if (Pfn1->u3.e2.ReferenceCount == 1)
1412     {
1413         /* It better not be valid */
1414         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1415
1416         /* Is it a prototype PTE? */
1417         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1418             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1419         {
1420             /* FIXME: We should return commit */
1421             DPRINT1("Not returning commit for prototype PTE\n");
1422         }
1423
1424         /* Update the counter */
1425         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1426     }
1427 }
1428
1429 //
1430 // Drops a locked page and dereferences it
1431 //
1432 FORCEINLINE
1433 VOID
1434 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1435 {
1436     USHORT RefCount, OldRefCount;
1437     PFN_NUMBER PageFrameIndex;
1438
1439     /* Loop while we decrement the page successfully */
1440     do
1441     {
1442         /* There should be at least one reference */
1443         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1444         ASSERT(OldRefCount != 0);
1445
1446         /* Are we the last one */
1447         if (OldRefCount == 1)
1448         {
1449             /* The page shoudln't be shared not active at this point */
1450             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1451             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1452             ASSERT(Pfn1->u2.ShareCount == 0);
1453
1454             /* Is it a prototype PTE? */
1455             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1456                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1457             {
1458                 /* FIXME: We should return commit */
1459                 DPRINT1("Not returning commit for prototype PTE\n");
1460             }
1461
1462             /* Update the counter, and drop a reference the long way */
1463             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1464             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1465             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1466             return;
1467         }
1468
1469         /* Drop a reference the short way, and that's it */
1470         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1471                                                 OldRefCount - 1,
1472                                                 OldRefCount);
1473         ASSERT(RefCount != 0);
1474     } while (OldRefCount != RefCount);
1475
1476     /* If we got here, there should be more than one reference */
1477     ASSERT(RefCount > 1);
1478     if (RefCount == 2)
1479     {
1480         /* Is it still being shared? */
1481         if (Pfn1->u2.ShareCount >= 1)
1482         {
1483             /* Then it should be valid */
1484             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1485
1486             /* Is it a prototype PTE? */
1487             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1488                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1489             {
1490                 /* We don't handle ethis */
1491                 ASSERT(FALSE);
1492             }
1493
1494             /* Update the counter */
1495             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1496         }
1497     }
1498 }
1499
1500 //
1501 // References a locked page and updates the counter
1502 // Used in MmProbeAndLockPages to handle different edge cases
1503 //
1504 FORCEINLINE
1505 VOID
1506 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1507 {
1508     USHORT RefCount, OldRefCount;
1509
1510     /* Sanity check */
1511     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1512
1513     /* Does ARM3 own the page? */
1514     if (MI_IS_ROS_PFN(Pfn1))
1515     {
1516         /* ReactOS Mm doesn't track share count */
1517         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1518     }
1519     else
1520     {
1521         /* On ARM3 pages, we should see a valid share count */
1522         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1523
1524         /* Is it a prototype PTE? */
1525         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1526             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1527         {
1528             /* FIXME: We should charge commit */
1529             DPRINT1("Not charging commit for prototype PTE\n");
1530         }
1531     }
1532
1533     /* More locked pages! */
1534     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1535
1536     /* Loop trying to update the reference count */
1537     do
1538     {
1539         /* Get the current reference count, make sure it's valid */
1540         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1541         ASSERT(OldRefCount != 0);
1542         ASSERT(OldRefCount < 2500);
1543
1544         /* Bump it up by one */
1545         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1546                                                 OldRefCount + 1,
1547                                                 OldRefCount);
1548         ASSERT(RefCount != 0);
1549     } while (OldRefCount != RefCount);
1550
1551     /* Was this the first lock attempt? If not, undo our bump */
1552     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1553 }
1554
1555 //
1556 // References a locked page and updates the counter
1557 // Used in all other cases except MmProbeAndLockPages
1558 //
1559 FORCEINLINE
1560 VOID
1561 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1562 {
1563     USHORT NewRefCount;
1564
1565     /* Is it a prototype PTE? */
1566     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1567         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1568     {
1569         /* FIXME: We should charge commit */
1570         DPRINT1("Not charging commit for prototype PTE\n");
1571     }
1572
1573     /* More locked pages! */
1574     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1575
1576     /* Update the reference count */
1577     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1578     if (NewRefCount == 2)
1579     {
1580         /* Is it locked or shared? */
1581         if (Pfn1->u2.ShareCount)
1582         {
1583             /* It's shared, so make sure it's active */
1584             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1585         }
1586         else
1587         {
1588             /* It's locked, so we shouldn't lock again */
1589             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1590         }
1591     }
1592     else
1593     {
1594         /* Someone had already locked the page, so undo our bump */
1595         ASSERT(NewRefCount < 2500);
1596         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1597     }
1598 }
1599
1600 //
1601 // References a locked page and updates the counter
1602 // Used in all other cases except MmProbeAndLockPages
1603 //
1604 FORCEINLINE
1605 VOID
1606 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1607 {
1608     USHORT NewRefCount;
1609
1610     /* Make sure the page isn't used yet */
1611     ASSERT(Pfn1->u2.ShareCount == 0);
1612     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1613
1614     /* Is it a prototype PTE? */
1615     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1616         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1617     {
1618         /* FIXME: We should charge commit */
1619         DPRINT1("Not charging commit for prototype PTE\n");
1620     }
1621
1622     /* More locked pages! */
1623     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1624
1625     /* Update the reference count */
1626     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1627     if (NewRefCount != 1)
1628     {
1629         /* Someone had already locked the page, so undo our bump */
1630         ASSERT(NewRefCount < 2500);
1631         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1632     }
1633 }
1634
1635 FORCEINLINE
1636 VOID
1637 MiIncrementPageTableReferences(IN PVOID Address)
1638 {
1639     PUSHORT RefCount;
1640
1641     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1642
1643     *RefCount += 1;
1644     ASSERT(*RefCount <= PTE_PER_PAGE);
1645 }
1646
1647 FORCEINLINE
1648 VOID
1649 MiDecrementPageTableReferences(IN PVOID Address)
1650 {
1651     PUSHORT RefCount;
1652
1653     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1654
1655     *RefCount -= 1;
1656     ASSERT(*RefCount < PTE_PER_PAGE);
1657 }
1658
1659 FORCEINLINE
1660 USHORT
1661 MiQueryPageTableReferences(IN PVOID Address)
1662 {
1663     PUSHORT RefCount;
1664
1665     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1666
1667     return *RefCount;
1668 }
1669
1670 BOOLEAN
1671 NTAPI
1672 MmArmInitSystem(
1673     IN ULONG Phase,
1674     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1675 );
1676
1677 VOID
1678 NTAPI
1679 MiInitializeSessionSpaceLayout(VOID);
1680
1681 NTSTATUS
1682 NTAPI
1683 MiInitMachineDependent(
1684     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1685 );
1686
1687 VOID
1688 NTAPI
1689 MiComputeColorInformation(
1690     VOID
1691 );
1692
1693 VOID
1694 NTAPI
1695 MiMapPfnDatabase(
1696     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1697 );
1698
1699 VOID
1700 NTAPI
1701 MiInitializeColorTables(
1702     VOID
1703 );
1704
1705 VOID
1706 NTAPI
1707 MiInitializePfnDatabase(
1708     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1709 );
1710
1711 VOID
1712 NTAPI
1713 MiInitializeSessionWsSupport(
1714     VOID
1715 );
1716
1717 VOID
1718 NTAPI
1719 MiInitializeSessionIds(
1720     VOID
1721 );
1722
1723 BOOLEAN
1724 NTAPI
1725 MiInitializeMemoryEvents(
1726     VOID
1727 );
1728
1729 PFN_NUMBER
1730 NTAPI
1731 MxGetNextPage(
1732     IN PFN_NUMBER PageCount
1733 );
1734
1735 PPHYSICAL_MEMORY_DESCRIPTOR
1736 NTAPI
1737 MmInitializeMemoryLimits(
1738     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1739     IN PBOOLEAN IncludeType
1740 );
1741
1742 PFN_NUMBER
1743 NTAPI
1744 MiPagesInLoaderBlock(
1745     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1746     IN PBOOLEAN IncludeType
1747 );
1748
1749 VOID
1750 FASTCALL
1751 MiSyncARM3WithROS(
1752     IN PVOID AddressStart,
1753     IN PVOID AddressEnd
1754 );
1755
1756 NTSTATUS
1757 NTAPI
1758 MiRosProtectVirtualMemory(
1759     IN PEPROCESS Process,
1760     IN OUT PVOID *BaseAddress,
1761     IN OUT PSIZE_T NumberOfBytesToProtect,
1762     IN ULONG NewAccessProtection,
1763     OUT PULONG OldAccessProtection OPTIONAL
1764 );
1765
1766 NTSTATUS
1767 NTAPI
1768 MmArmAccessFault(
1769     IN ULONG FaultCode,
1770     IN PVOID Address,
1771     IN KPROCESSOR_MODE Mode,
1772     IN PVOID TrapInformation
1773 );
1774
1775 NTSTATUS
1776 FASTCALL
1777 MiCheckPdeForPagedPool(
1778     IN PVOID Address
1779 );
1780
1781 VOID
1782 NTAPI
1783 MiInitializeNonPagedPool(
1784     VOID
1785 );
1786
1787 VOID
1788 NTAPI
1789 MiInitializeNonPagedPoolThresholds(
1790     VOID
1791 );
1792
1793 VOID
1794 NTAPI
1795 MiInitializePoolEvents(
1796     VOID
1797 );
1798
1799 VOID                      //
1800 NTAPI                     //
1801 InitializePool(           //
1802     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1803     IN ULONG Threshold    //
1804 );                        //
1805
1806 // FIXFIX: THIS ONE TOO
1807 VOID
1808 NTAPI
1809 INIT_FUNCTION
1810 ExInitializePoolDescriptor(
1811     IN PPOOL_DESCRIPTOR PoolDescriptor,
1812     IN POOL_TYPE PoolType,
1813     IN ULONG PoolIndex,
1814     IN ULONG Threshold,
1815     IN PVOID PoolLock
1816 );
1817
1818 NTSTATUS
1819 NTAPI
1820 MiInitializeSessionPool(
1821     VOID
1822 );
1823
1824 VOID
1825 NTAPI
1826 MiInitializeSystemPtes(
1827     IN PMMPTE StartingPte,
1828     IN ULONG NumberOfPtes,
1829     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1830 );
1831
1832 PMMPTE
1833 NTAPI
1834 MiReserveSystemPtes(
1835     IN ULONG NumberOfPtes,
1836     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1837 );
1838
1839 VOID
1840 NTAPI
1841 MiReleaseSystemPtes(
1842     IN PMMPTE StartingPte,
1843     IN ULONG NumberOfPtes,
1844     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1845 );
1846
1847
1848 PFN_NUMBER
1849 NTAPI
1850 MiFindContiguousPages(
1851     IN PFN_NUMBER LowestPfn,
1852     IN PFN_NUMBER HighestPfn,
1853     IN PFN_NUMBER BoundaryPfn,
1854     IN PFN_NUMBER SizeInPages,
1855     IN MEMORY_CACHING_TYPE CacheType
1856 );
1857
1858 PVOID
1859 NTAPI
1860 MiCheckForContiguousMemory(
1861     IN PVOID BaseAddress,
1862     IN PFN_NUMBER BaseAddressPages,
1863     IN PFN_NUMBER SizeInPages,
1864     IN PFN_NUMBER LowestPfn,
1865     IN PFN_NUMBER HighestPfn,
1866     IN PFN_NUMBER BoundaryPfn,
1867     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1868 );
1869
1870 PMDL
1871 NTAPI
1872 MiAllocatePagesForMdl(
1873     IN PHYSICAL_ADDRESS LowAddress,
1874     IN PHYSICAL_ADDRESS HighAddress,
1875     IN PHYSICAL_ADDRESS SkipBytes,
1876     IN SIZE_T TotalBytes,
1877     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1878     IN ULONG Flags
1879 );
1880
1881 VOID
1882 NTAPI
1883 MiInsertPageInList(
1884     IN PMMPFNLIST ListHead,
1885     IN PFN_NUMBER PageFrameIndex
1886 );
1887
1888 VOID
1889 NTAPI
1890 MiUnlinkFreeOrZeroedPage(
1891     IN PMMPFN Entry
1892 );
1893
1894 VOID
1895 NTAPI
1896 MiUnlinkPageFromList(
1897     IN PMMPFN Pfn
1898 );
1899
1900 VOID
1901 NTAPI
1902 MiInitializePfn(
1903     IN PFN_NUMBER PageFrameIndex,
1904     IN PMMPTE PointerPte,
1905     IN BOOLEAN Modified
1906 );
1907
1908 NTSTATUS
1909 NTAPI
1910 MiInitializeAndChargePfn(
1911     OUT PPFN_NUMBER PageFrameIndex,
1912     IN PMMPDE PointerPde,
1913     IN PFN_NUMBER ContainingPageFrame,
1914     IN BOOLEAN SessionAllocation
1915 );
1916
1917 VOID
1918 NTAPI
1919 MiInitializePfnAndMakePteValid(
1920     IN PFN_NUMBER PageFrameIndex,
1921     IN PMMPTE PointerPte,
1922     IN MMPTE TempPte
1923 );
1924
1925 VOID
1926 NTAPI
1927 MiInitializePfnForOtherProcess(
1928     IN PFN_NUMBER PageFrameIndex,
1929     IN PVOID PteAddress,
1930     IN PFN_NUMBER PteFrame
1931 );
1932
1933 VOID
1934 NTAPI
1935 MiDecrementShareCount(
1936     IN PMMPFN Pfn1,
1937     IN PFN_NUMBER PageFrameIndex
1938 );
1939
1940 PFN_NUMBER
1941 NTAPI
1942 MiRemoveAnyPage(
1943     IN ULONG Color
1944 );
1945
1946 PFN_NUMBER
1947 NTAPI
1948 MiRemoveZeroPage(
1949     IN ULONG Color
1950 );
1951
1952 VOID
1953 NTAPI
1954 MiZeroPhysicalPage(
1955     IN PFN_NUMBER PageFrameIndex
1956 );
1957
1958 VOID
1959 NTAPI
1960 MiInsertPageInFreeList(
1961     IN PFN_NUMBER PageFrameIndex
1962 );
1963
1964 PFN_COUNT
1965 NTAPI
1966 MiDeleteSystemPageableVm(
1967     IN PMMPTE PointerPte,
1968     IN PFN_NUMBER PageCount,
1969     IN ULONG Flags,
1970     OUT PPFN_NUMBER ValidPages
1971 );
1972
1973 ULONG
1974 NTAPI
1975 MiGetPageProtection(
1976     IN PMMPTE PointerPte
1977 );
1978
1979 PLDR_DATA_TABLE_ENTRY
1980 NTAPI
1981 MiLookupDataTableEntry(
1982     IN PVOID Address
1983 );
1984
1985 VOID
1986 NTAPI
1987 MiInitializeDriverLargePageList(
1988     VOID
1989 );
1990
1991 VOID
1992 NTAPI
1993 MiInitializeLargePageSupport(
1994     VOID
1995 );
1996
1997 VOID
1998 NTAPI
1999 MiSyncCachedRanges(
2000     VOID
2001 );
2002
2003 BOOLEAN
2004 NTAPI
2005 MiIsPfnInUse(
2006     IN PMMPFN Pfn1
2007 );
2008
2009 PMMVAD
2010 NTAPI
2011 MiLocateAddress(
2012     IN PVOID VirtualAddress
2013 );
2014
2015 TABLE_SEARCH_RESULT
2016 NTAPI
2017 MiCheckForConflictingNode(
2018     IN ULONG_PTR StartVpn,
2019     IN ULONG_PTR EndVpn,
2020     IN PMM_AVL_TABLE Table,
2021     OUT PMMADDRESS_NODE *NodeOrParent
2022 );
2023
2024 TABLE_SEARCH_RESULT
2025 NTAPI
2026 MiFindEmptyAddressRangeDownTree(
2027     IN SIZE_T Length,
2028     IN ULONG_PTR BoundaryAddress,
2029     IN ULONG_PTR Alignment,
2030     IN PMM_AVL_TABLE Table,
2031     OUT PULONG_PTR Base,
2032     OUT PMMADDRESS_NODE *Parent
2033 );
2034
2035 NTSTATUS
2036 NTAPI
2037 MiFindEmptyAddressRangeDownBasedTree(
2038     IN SIZE_T Length,
2039     IN ULONG_PTR BoundaryAddress,
2040     IN ULONG_PTR Alignment,
2041     IN PMM_AVL_TABLE Table,
2042     OUT PULONG_PTR Base
2043 );
2044
2045 TABLE_SEARCH_RESULT
2046 NTAPI
2047 MiFindEmptyAddressRangeInTree(
2048     IN SIZE_T Length,
2049     IN ULONG_PTR Alignment,
2050     IN PMM_AVL_TABLE Table,
2051     OUT PMMADDRESS_NODE *PreviousVad,
2052     OUT PULONG_PTR Base
2053 );
2054
2055 NTSTATUS
2056 NTAPI
2057 MiCheckSecuredVad(
2058     IN PMMVAD Vad,
2059     IN PVOID Base,
2060     IN SIZE_T Size,
2061     IN ULONG ProtectionMask
2062 );
2063
2064 VOID
2065 NTAPI
2066 MiInsertVad(
2067     _Inout_ PMMVAD Vad,
2068     _Inout_ PMM_AVL_TABLE VadRoot);
2069
2070 NTSTATUS
2071 NTAPI
2072 MiInsertVadEx(
2073     _Inout_ PMMVAD Vad,
2074     _In_ ULONG_PTR *BaseAddress,
2075     _In_ SIZE_T ViewSize,
2076     _In_ ULONG_PTR HighestAddress,
2077     _In_ ULONG_PTR Alignment,
2078     _In_ ULONG AllocationType);
2079
2080 VOID
2081 NTAPI
2082 MiInsertBasedSection(
2083     IN PSECTION Section
2084 );
2085
2086 NTSTATUS
2087 NTAPI
2088 MiUnmapViewOfSection(
2089     IN PEPROCESS Process,
2090     IN PVOID BaseAddress,
2091     IN ULONG Flags
2092 );
2093
2094 NTSTATUS
2095 NTAPI
2096 MiRosUnmapViewOfSection(
2097     IN PEPROCESS Process,
2098     IN PVOID BaseAddress,
2099     IN BOOLEAN SkipDebuggerNotify
2100 );
2101
2102 VOID
2103 NTAPI
2104 MiInsertNode(
2105     IN PMM_AVL_TABLE Table,
2106     IN PMMADDRESS_NODE NewNode,
2107     PMMADDRESS_NODE Parent,
2108     TABLE_SEARCH_RESULT Result
2109 );
2110
2111 VOID
2112 NTAPI
2113 MiRemoveNode(
2114     IN PMMADDRESS_NODE Node,
2115     IN PMM_AVL_TABLE Table
2116 );
2117
2118 PMMADDRESS_NODE
2119 NTAPI
2120 MiGetPreviousNode(
2121     IN PMMADDRESS_NODE Node
2122 );
2123
2124 PMMADDRESS_NODE
2125 NTAPI
2126 MiGetNextNode(
2127     IN PMMADDRESS_NODE Node
2128 );
2129
2130 BOOLEAN
2131 NTAPI
2132 MiInitializeSystemSpaceMap(
2133     IN PMMSESSION InputSession OPTIONAL
2134 );
2135
2136 VOID
2137 NTAPI
2138 MiSessionRemoveProcess(
2139     VOID
2140 );
2141
2142 VOID
2143 NTAPI
2144 MiReleaseProcessReferenceToSessionDataPage(
2145     IN PMM_SESSION_SPACE SessionGlobal
2146 );
2147
2148 VOID
2149 NTAPI
2150 MiSessionAddProcess(
2151     IN PEPROCESS NewProcess
2152 );
2153
2154 NTSTATUS
2155 NTAPI
2156 MiSessionCommitPageTables(
2157     IN PVOID StartVa,
2158     IN PVOID EndVa
2159 );
2160
2161 ULONG
2162 NTAPI
2163 MiMakeProtectionMask(
2164     IN ULONG Protect
2165 );
2166
2167 VOID
2168 NTAPI
2169 MiDeleteVirtualAddresses(
2170     IN ULONG_PTR Va,
2171     IN ULONG_PTR EndingAddress,
2172     IN PMMVAD Vad
2173 );
2174
2175 VOID
2176 NTAPI
2177 MiDeletePte(
2178     IN PMMPTE PointerPte,
2179     IN PVOID VirtualAddress,
2180     IN PEPROCESS CurrentProcess,
2181     IN PMMPTE PrototypePte
2182 );
2183
2184 ULONG
2185 NTAPI
2186 MiMakeSystemAddressValid(
2187     IN PVOID PageTableVirtualAddress,
2188     IN PEPROCESS CurrentProcess
2189 );
2190
2191 ULONG
2192 NTAPI
2193 MiMakeSystemAddressValidPfn(
2194     IN PVOID VirtualAddress,
2195     IN KIRQL OldIrql
2196 );
2197
2198 VOID
2199 NTAPI
2200 MiRemoveMappedView(
2201     IN PEPROCESS CurrentProcess,
2202     IN PMMVAD Vad
2203 );
2204
2205 PSUBSECTION
2206 NTAPI
2207 MiLocateSubsection(
2208     IN PMMVAD Vad,
2209     IN ULONG_PTR Vpn
2210 );
2211
2212 VOID
2213 NTAPI
2214 MiDeleteARM3Section(
2215     PVOID ObjectBody
2216 );
2217
2218 NTSTATUS
2219 NTAPI
2220 MiQueryMemorySectionName(
2221     IN HANDLE ProcessHandle,
2222     IN PVOID BaseAddress,
2223     OUT PVOID MemoryInformation,
2224     IN SIZE_T MemoryInformationLength,
2225     OUT PSIZE_T ReturnLength
2226 );
2227
2228 NTSTATUS
2229 NTAPI
2230 MiRosUnmapViewInSystemSpace(
2231     IN PVOID MappedBase
2232 );
2233
2234 POOL_TYPE
2235 NTAPI
2236 MmDeterminePoolType(
2237     IN PVOID PoolAddress
2238 );
2239
2240 VOID
2241 NTAPI
2242 MiMakePdeExistAndMakeValid(
2243     IN PMMPDE PointerPde,
2244     IN PEPROCESS TargetProcess,
2245     IN KIRQL OldIrql
2246 );
2247
2248 //
2249 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2250 // free pages are available. In some scenarios, we don't/can't run that piece of
2251 // code and would rather only have a real zero page. If we can't have a zero page,
2252 // then we'd like to have our own code to grab a free page and zero it out, by
2253 // using MiRemoveAnyPage. This macro implements this.
2254 //
2255 FORCEINLINE
2256 PFN_NUMBER
2257 MiRemoveZeroPageSafe(IN ULONG Color)
2258 {
2259     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2260     return 0;
2261 }
2262
2263 #if (_MI_PAGING_LEVELS == 2)
2264 FORCEINLINE
2265 BOOLEAN
2266 MiSynchronizeSystemPde(PMMPDE PointerPde)
2267 {
2268     MMPDE SystemPde;
2269     ULONG Index;
2270
2271     /* Get the Index from the PDE */
2272     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2273
2274     /* Copy the PDE from the double-mapped system page directory */
2275     SystemPde = MmSystemPagePtes[Index];
2276     *PointerPde = SystemPde;
2277
2278     /* Make sure we re-read the PDE and PTE */
2279     KeMemoryBarrierWithoutFence();
2280
2281     /* Return, if we had success */
2282     return SystemPde.u.Hard.Valid != 0;
2283 }
2284 #endif
2285
2286 /* EOF */