ntoskrnl/mm/ARM3/miarm.h

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
   5  * PURPOSE:         ARM Memory Manager Header
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 #pragma once
  10
  11 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
  12
  13 /* Make the code cleaner with some definitions for size multiples */
  14 #define _1KB (1024u)
  15 #define _1MB (1024 * _1KB)
  16 #define _1GB (1024 * _1MB)
  17
  18 /* Everyone loves 64K */
  19 #define _64K (64 * _1KB)
  20
  21 /* Area mapped by a PDE */
  22 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
  23
  24 /* Size of a page table */
  25 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
  26
  27 /* Size of a page directory */
  28 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
  29
  30 /* Stop using these! */
  31 #define PD_COUNT  PPE_PER_PAGE
  32 #define PDE_COUNT PDE_PER_PAGE
  33 #define PTE_COUNT PTE_PER_PAGE
  34
  35 /* Size of all page directories for a process */
  36 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
  37 #ifdef _M_IX86
  38 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
  39 #endif
  40
  41 //
  42 // Protection Bits part of the internal memory manager Protection Mask, from:
  43 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
  44 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
  45 // and public assertions.
  46 //
  47 #define MM_ZERO_ACCESS         0
  48 #define MM_READONLY            1
  49 #define MM_EXECUTE             2
  50 #define MM_EXECUTE_READ        3
  51 #define MM_READWRITE           4
  52 #define MM_WRITECOPY           5
  53 #define MM_EXECUTE_READWRITE   6
  54 #define MM_EXECUTE_WRITECOPY   7
  55 #define MM_PROTECT_ACCESS      7
  56
  57 //
  58 // These are flags on top of the actual protection mask
  59 //
  60 #define MM_NOCACHE            0x08
  61 #define MM_GUARDPAGE          0x10
  62 #define MM_WRITECOMBINE       0x18
  63 #define MM_PROTECT_SPECIAL    0x18
  64
  65 //
  66 // These are special cases
  67 //
  68 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
  69 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
  70 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
  71 #define MM_INVALID_PROTECTION  0xFFFFFFFF
  72
  73 //
  74 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
  75 // The Memory Manager's definition define the attributes that must be preserved
  76 // and these PTE definitions describe the attributes in the hardware sense. This
  77 // helps deal with hardware differences between the actual boolean expression of
  78 // the argument.
  79 //
  80 // For example, in the logical attributes, we want to express read-only as a flag
  81 // but on x86, it is writability that must be set. On the other hand, on x86, just
  82 // like in the kernel, it is disabling the caches that requires a special flag,
  83 // while on certain architectures such as ARM, it is enabling the cache which
  84 // requires a flag.
  85 //
  86 #if defined(_M_IX86)
  87 //
  88 // Access Flags
  89 //
  90 #define PTE_READONLY            0 // Doesn't exist on x86
  91 #define PTE_EXECUTE             0 // Not worrying about NX yet
  92 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
  93 #define PTE_READWRITE           0x2
  94 #define PTE_WRITECOPY           0x200
  95 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
  96 #define PTE_EXECUTE_WRITECOPY   0x200
  97 #define PTE_PROTOTYPE           0x400
  98
  99 //
 100 // State Flags
 101 //
 102 #define PTE_VALID               0x1
 103 #define PTE_ACCESSED            0x20
 104 #define PTE_DIRTY               0x40
 105
 106 //
 107 // Cache flags
 108 //
 109 #define PTE_ENABLE_CACHE        0
 110 #define PTE_DISABLE_CACHE       0x10
 111 #define PTE_WRITECOMBINED_CACHE 0x10
 112 #define PTE_PROTECT_MASK        0x612
 113 #elif defined(_M_AMD64)
 114 //
 115 // Access Flags
 116 //
 117 #define PTE_READONLY            0x8000000000000000ULL
 118 #define PTE_EXECUTE             0x0000000000000000ULL
 119 #define PTE_EXECUTE_READ        PTE_EXECUTE /* EXECUTE implies READ on x64 */
 120 #define PTE_READWRITE           0x8000000000000002ULL
 121 #define PTE_WRITECOPY           0x8000000000000200ULL
 122 #define PTE_EXECUTE_READWRITE   0x0000000000000002ULL
 123 #define PTE_EXECUTE_WRITECOPY   0x0000000000000200ULL
 124 #define PTE_PROTOTYPE           0x0000000000000400ULL
 125
 126 //
 127 // State Flags
 128 //
 129 #define PTE_VALID               0x0000000000000001ULL
 130 #define PTE_ACCESSED            0x0000000000000020ULL
 131 #define PTE_DIRTY               0x0000000000000040ULL
 132
 133 //
 134 // Cache flags
 135 //
 136 #define PTE_ENABLE_CACHE        0x0000000000000000ULL
 137 #define PTE_DISABLE_CACHE       0x0000000000000010ULL
 138 #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL
 139 #define PTE_PROTECT_MASK        0x8000000000000612ULL
 140 #elif defined(_M_ARM)
 141 #define PTE_READONLY            0x200
 142 #define PTE_EXECUTE             0 // Not worrying about NX yet
 143 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 144 #define PTE_READWRITE           0 // Doesn't exist on ARM
 145 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
 146 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
 147 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
 148 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
 149
 150 //
 151 // Cache flags
 152 //
 153 #define PTE_ENABLE_CACHE        0
 154 #define PTE_DISABLE_CACHE       0x10
 155 #define PTE_WRITECOMBINED_CACHE 0x10
 156 #define PTE_PROTECT_MASK        0x610
 157 #else
 158 #error Define these please!
 159 #endif
 160
 161 //
 162 // Mask for image section page protection
 163 //
 164 #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE)
 165
 166 extern const ULONG_PTR MmProtectToPteMask[32];
 167 extern const ULONG MmProtectToValue[32];
 168
 169 //
 170 // Assertions for session images, addresses, and PTEs
 171 //
 172 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
 173     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
 174
 175 #define MI_IS_SESSION_ADDRESS(Address) \
 176     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
 177
 178 #define MI_IS_SESSION_PTE(Pte) \
 179     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
 180
 181 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
 182     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
 183
 184 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
 185     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
 186
 187 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
 188     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
 189
 190 //
 191 // Creates a software PTE with the given protection
 192 //
 193 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
 194
 195 //
 196 // Marks a PTE as deleted
 197 //
 198 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
 199 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
 200
 201 //
 202 // Special values for LoadedImports
 203 //
 204 #ifdef _WIN64
 205 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFFFFFFFFFEULL
 206 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFFFFFFFFFFULL
 207 #else
 208 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
 209 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
 210 #endif
 211 #define MM_SYSLDR_SINGLE_ENTRY 0x1
 212
 213 //
 214 // Number of initial session IDs
 215 //
 216 #define MI_INITIAL_SESSION_IDS  64
 217
 218 #if defined(_M_IX86) || defined(_M_ARM)
 219 //
 220 // PFN List Sentinel
 221 //
 222 #define LIST_HEAD 0xFFFFFFFF
 223
 224 //
 225 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
 226 // we need a manual definition suited to the number of bits in the PteFrame.
 227 // This is used as a LIST_HEAD for the colored list
 228 //
 229 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
 230 #elif defined(_M_AMD64)
 231 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
 232 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
 233 #else
 234 #error Define these please!
 235 #endif
 236
 237 //
 238 // Special IRQL value (found in assertions)
 239 //
 240 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 241
 242 //
 243 // Returns the color of a page
 244 //
 245 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
 246 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
 247 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
 248
 249 //
 250 // Prototype PTEs that don't yet have a pagefile association
 251 //
 252 #ifdef _WIN64
 253 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
 254 #else
 255 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
 256 #endif
 257
 258 //
 259 // Number of session data and tag pages
 260 //
 261 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 262 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 263
 264 //
 265 // Used by MiCheckSecuredVad
 266 //
 267 #define MM_READ_WRITE_ALLOWED   11
 268 #define MM_READ_ONLY_ALLOWED    10
 269 #define MM_NO_ACCESS_ALLOWED    01
 270 #define MM_DELETE_CHECK         85
 271
 272 //
 273 // System views are binned into 64K chunks
 274 //
 275 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
 276
 277 //
 278 // FIXFIX: These should go in ex.h after the pool merge
 279 //
 280 #ifdef _WIN64
 281 #define POOL_BLOCK_SIZE 16
 282 #else
 283 #define POOL_BLOCK_SIZE  8
 284 #endif
 285 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
 286 #define BASE_POOL_TYPE_MASK 1
 287 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
 288
 289 //
 290 // Pool debugging/analysis/tracing flags
 291 //
 292 #define POOL_FLAG_CHECK_TIMERS 0x1
 293 #define POOL_FLAG_CHECK_WORKERS 0x2
 294 #define POOL_FLAG_CHECK_RESOURCES 0x4
 295 #define POOL_FLAG_VERIFIER 0x8
 296 #define POOL_FLAG_CHECK_DEADLOCK 0x10
 297 #define POOL_FLAG_SPECIAL_POOL 0x20
 298 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
 299 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
 300
 301 //
 302 // BAD_POOL_HEADER codes during pool bugcheck
 303 //
 304 #define POOL_CORRUPTED_LIST 3
 305 #define POOL_SIZE_OR_INDEX_MISMATCH 5
 306 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
 307 #define POOL_HEADER_NOT_ALIGNED 7
 308 #define POOL_HEADER_IS_ZERO 8
 309 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
 310 #define POOL_ENTRY_NOT_FOUND 10
 311
 312 //
 313 // BAD_POOL_CALLER codes during pool bugcheck
 314 //
 315 #define POOL_ENTRY_CORRUPTED 1
 316 #define POOL_ENTRY_ALREADY_FREE 6
 317 #define POOL_ENTRY_NOT_ALLOCATED 7
 318 #define POOL_ALLOC_IRQL_INVALID 8
 319 #define POOL_FREE_IRQL_INVALID 9
 320 #define POOL_BILLED_PROCESS_INVALID 13
 321 #define POOL_HEADER_SIZE_INVALID 32
 322
 323 typedef struct _POOL_DESCRIPTOR
 324 {
 325     POOL_TYPE PoolType;
 326     ULONG PoolIndex;
 327     ULONG RunningAllocs;
 328     ULONG RunningDeAllocs;
 329     ULONG TotalPages;
 330     ULONG TotalBigPages;
 331     ULONG Threshold;
 332     PVOID LockAddress;
 333     PVOID PendingFrees;
 334     LONG PendingFreeDepth;
 335     SIZE_T TotalBytes;
 336     SIZE_T Spare0;
 337     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
 338 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 339
 340 typedef struct _POOL_HEADER
 341 {
 342     union
 343     {
 344         struct
 345         {
 346 #ifdef _WIN64
 347             USHORT PreviousSize:8;
 348             USHORT PoolIndex:8;
 349             USHORT BlockSize:8;
 350             USHORT PoolType:8;
 351 #else
 352             USHORT PreviousSize:9;
 353             USHORT PoolIndex:7;
 354             USHORT BlockSize:9;
 355             USHORT PoolType:7;
 356 #endif
 357         };
 358         ULONG Ulong1;
 359     };
 360 #ifdef _WIN64
 361     ULONG PoolTag;
 362 #endif
 363     union
 364     {
 365 #ifdef _WIN64
 366         PEPROCESS ProcessBilled;
 367 #else
 368         ULONG PoolTag;
 369 #endif
 370         struct
 371         {
 372             USHORT AllocatorBackTraceIndex;
 373             USHORT PoolTagHash;
 374         };
 375     };
 376 } POOL_HEADER, *PPOOL_HEADER;
 377
 378 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
 379 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
 380
 381 typedef struct _POOL_TRACKER_TABLE
 382 {
 383     ULONG Key;
 384     LONG NonPagedAllocs;
 385     LONG NonPagedFrees;
 386     SIZE_T NonPagedBytes;
 387     LONG PagedAllocs;
 388     LONG PagedFrees;
 389     SIZE_T PagedBytes;
 390 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
 391
 392 typedef struct _POOL_TRACKER_BIG_PAGES
 393 {
 394     PVOID Va;
 395     ULONG Key;
 396     ULONG NumberOfPages;
 397     PVOID QuotaObject;
 398 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
 399
 400 extern ULONG ExpNumberOfPagedPools;
 401 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
 402 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
 403 extern PPOOL_TRACKER_TABLE PoolTrackTable;
 404
 405 //
 406 // END FIXFIX
 407 //
 408
 409 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
 410 {
 411     LIST_ENTRY Links;
 412     UNICODE_STRING BaseName;
 413 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
 414
 415 typedef enum _MMSYSTEM_PTE_POOL_TYPE
 416 {
 417     SystemPteSpace,
 418     NonPagedPoolExpansion,
 419     MaximumPtePoolTypes
 420 } MMSYSTEM_PTE_POOL_TYPE;
 421
 422 typedef enum _MI_PFN_CACHE_ATTRIBUTE
 423 {
 424     MiNonCached,
 425     MiCached,
 426     MiWriteCombined,
 427     MiNotMapped
 428 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
 429
 430 typedef struct _PHYSICAL_MEMORY_RUN
 431 {
 432     PFN_NUMBER BasePage;
 433     PFN_NUMBER PageCount;
 434 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
 435
 436 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
 437 {
 438     ULONG NumberOfRuns;
 439     PFN_NUMBER NumberOfPages;
 440     PHYSICAL_MEMORY_RUN Run[1];
 441 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
 442
 443 typedef struct _MMCOLOR_TABLES
 444 {
 445     PFN_NUMBER Flink;
 446     PVOID Blink;
 447     PFN_NUMBER Count;
 448 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 449
 450 typedef struct _MI_LARGE_PAGE_RANGES
 451 {
 452     PFN_NUMBER StartFrame;
 453     PFN_NUMBER LastFrame;
 454 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
 455
 456 typedef struct _MMVIEW
 457 {
 458     ULONG_PTR Entry;
 459     PCONTROL_AREA ControlArea;
 460 } MMVIEW, *PMMVIEW;
 461
 462 typedef struct _MMSESSION
 463 {
 464     KGUARDED_MUTEX SystemSpaceViewLock;
 465     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
 466     PCHAR SystemSpaceViewStart;
 467     PMMVIEW SystemSpaceViewTable;
 468     ULONG SystemSpaceHashSize;
 469     ULONG SystemSpaceHashEntries;
 470     ULONG SystemSpaceHashKey;
 471     ULONG BitmapFailures;
 472     PRTL_BITMAP SystemSpaceBitMap;
 473 } MMSESSION, *PMMSESSION;
 474
 475 typedef struct _MM_SESSION_SPACE_FLAGS
 476 {
 477     ULONG Initialized:1;
 478     ULONG DeletePending:1;
 479     ULONG Filler:30;
 480 } MM_SESSION_SPACE_FLAGS;
 481
 482 typedef struct _MM_SESSION_SPACE
 483 {
 484     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
 485     LONG ReferenceCount;
 486     union
 487     {
 488         ULONG LongFlags;
 489         MM_SESSION_SPACE_FLAGS Flags;
 490     } u;
 491     ULONG SessionId;
 492     LIST_ENTRY ProcessList;
 493     LARGE_INTEGER LastProcessSwappedOutTime;
 494     PFN_NUMBER SessionPageDirectoryIndex;
 495     SIZE_T NonPageablePages;
 496     SIZE_T CommittedPages;
 497     PVOID PagedPoolStart;
 498     PVOID PagedPoolEnd;
 499     PMMPDE PagedPoolBasePde;
 500     ULONG Color;
 501     LONG ResidentProcessCount;
 502     ULONG SessionPoolAllocationFailures[4];
 503     LIST_ENTRY ImageList;
 504     LCID LocaleId;
 505     ULONG AttachCount;
 506     KEVENT AttachEvent;
 507     PEPROCESS LastProcess;
 508     LONG ProcessReferenceToSession;
 509     LIST_ENTRY WsListEntry;
 510     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
 511     MMSESSION Session;
 512     KGUARDED_MUTEX PagedPoolMutex;
 513     MM_PAGED_POOL_INFO PagedPoolInfo;
 514     MMSUPPORT Vm;
 515     PMMWSLE Wsle;
 516     PDRIVER_UNLOAD Win32KDriverUnload;
 517     POOL_DESCRIPTOR PagedPool;
 518 #if defined (_M_AMD64)
 519     MMPDE PageDirectory;
 520 #else
 521     PMMPDE PageTables;
 522 #endif
 523 #if defined (_M_AMD64)
 524     PMMPTE SpecialPoolFirstPte;
 525     PMMPTE SpecialPoolLastPte;
 526     PMMPTE NextPdeForSpecialPoolExpansion;
 527     PMMPTE LastPdeForSpecialPoolExpansion;
 528     PFN_NUMBER SpecialPagesInUse;
 529 #endif
 530     LONG ImageLoadingCount;
 531 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
 532
 533 extern PMM_SESSION_SPACE MmSessionSpace;
 534 extern MMPTE HyperTemplatePte;
 535 extern MMPDE ValidKernelPde;
 536 extern MMPTE ValidKernelPte;
 537 extern MMPDE ValidKernelPdeLocal;
 538 extern MMPTE ValidKernelPteLocal;
 539 extern MMPDE DemandZeroPde;
 540 extern MMPTE DemandZeroPte;
 541 extern MMPTE PrototypePte;
 542 extern MMPTE MmDecommittedPte;
 543 extern BOOLEAN MmLargeSystemCache;
 544 extern BOOLEAN MmZeroPageFile;
 545 extern BOOLEAN MmProtectFreedNonPagedPool;
 546 extern BOOLEAN MmTrackLockedPages;
 547 extern BOOLEAN MmTrackPtes;
 548 extern BOOLEAN MmDynamicPfn;
 549 extern BOOLEAN MmMirroring;
 550 extern BOOLEAN MmMakeLowMemory;
 551 extern BOOLEAN MmEnforceWriteProtection;
 552 extern SIZE_T MmAllocationFragment;
 553 extern ULONG MmConsumedPoolPercentage;
 554 extern ULONG MmVerifyDriverBufferType;
 555 extern ULONG MmVerifyDriverLevel;
 556 extern WCHAR MmVerifyDriverBuffer[512];
 557 extern WCHAR MmLargePageDriverBuffer[512];
 558 extern LIST_ENTRY MiLargePageDriverList;
 559 extern BOOLEAN MiLargePageAllDrivers;
 560 extern ULONG MmVerifyDriverBufferLength;
 561 extern ULONG MmLargePageDriverBufferLength;
 562 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
 563 extern SIZE_T MmMaximumNonPagedPoolInBytes;
 564 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
 565 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
 566 extern PVOID MmNonPagedSystemStart;
 567 extern PVOID MmNonPagedPoolStart;
 568 extern PVOID MmNonPagedPoolExpansionStart;
 569 extern PVOID MmNonPagedPoolEnd;
 570 extern SIZE_T MmSizeOfPagedPoolInBytes;
 571 extern PVOID MmPagedPoolStart;
 572 extern PVOID MmPagedPoolEnd;
 573 extern PVOID MmSessionBase;
 574 extern SIZE_T MmSessionSize;
 575 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
 576 extern PMMPTE MiFirstReservedZeroingPte;
 577 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
 578 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
 579 extern SIZE_T MmBootImageSize;
 580 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
 581 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
 582 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
 583 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
 584 extern ULONG_PTR MxPfnAllocation;
 585 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
 586 extern RTL_BITMAP MiPfnBitMap;
 587 extern KGUARDED_MUTEX MmPagedPoolMutex;
 588 extern KGUARDED_MUTEX MmSectionCommitMutex;
 589 extern PVOID MmPagedPoolStart;
 590 extern PVOID MmPagedPoolEnd;
 591 extern PVOID MmNonPagedSystemStart;
 592 extern PVOID MiSystemViewStart;
 593 extern SIZE_T MmSystemViewSize;
 594 extern PVOID MmSessionBase;
 595 extern PVOID MiSessionSpaceEnd;
 596 extern PMMPTE MiSessionImagePteStart;
 597 extern PMMPTE MiSessionImagePteEnd;
 598 extern PMMPTE MiSessionBasePte;
 599 extern PMMPTE MiSessionLastPte;
 600 extern SIZE_T MmSizeOfPagedPoolInBytes;
 601 extern PMMPDE MmSystemPagePtes;
 602 extern PVOID MmSystemCacheStart;
 603 extern PVOID MmSystemCacheEnd;
 604 extern MMSUPPORT MmSystemCacheWs;
 605 extern SIZE_T MmAllocatedNonPagedPool;
 606 extern ULONG MmSpecialPoolTag;
 607 extern PVOID MmHyperSpaceEnd;
 608 extern PMMWSL MmSystemCacheWorkingSetList;
 609 extern SIZE_T MmMinimumNonPagedPoolSize;
 610 extern ULONG MmMinAdditionNonPagedPoolPerMb;
 611 extern SIZE_T MmDefaultMaximumNonPagedPool;
 612 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
 613 extern ULONG MmSecondaryColors;
 614 extern ULONG MmSecondaryColorMask;
 615 extern ULONG MmNumberOfSystemPtes;
 616 extern ULONG MmMaximumNonPagedPoolPercent;
 617 extern ULONG MmLargeStackSize;
 618 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
 619 extern MMPFNLIST MmStandbyPageListByPriority[8];
 620 extern ULONG MmProductType;
 621 extern MM_SYSTEMSIZE MmSystemSize;
 622 extern PKEVENT MiLowMemoryEvent;
 623 extern PKEVENT MiHighMemoryEvent;
 624 extern PKEVENT MiLowPagedPoolEvent;
 625 extern PKEVENT MiHighPagedPoolEvent;
 626 extern PKEVENT MiLowNonPagedPoolEvent;
 627 extern PKEVENT MiHighNonPagedPoolEvent;
 628 extern PFN_NUMBER MmLowMemoryThreshold;
 629 extern PFN_NUMBER MmHighMemoryThreshold;
 630 extern PFN_NUMBER MiLowPagedPoolThreshold;
 631 extern PFN_NUMBER MiHighPagedPoolThreshold;
 632 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
 633 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 634 extern PFN_NUMBER MmMinimumFreePages;
 635 extern PFN_NUMBER MmPlentyFreePages;
 636 extern SIZE_T MmMinimumStackCommitInBytes;
 637 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 638 extern PFN_NUMBER MmResidentAvailablePages;
 639 extern PFN_NUMBER MmResidentAvailableAtInit;
 640 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
 641 extern PFN_NUMBER MmTotalSystemDriverPages;
 642 extern ULONG MmCritsectTimeoutSeconds;
 643 extern PVOID MiSessionImageStart;
 644 extern PVOID MiSessionImageEnd;
 645 extern PMMPTE MiHighestUserPte;
 646 extern PMMPDE MiHighestUserPde;
 647 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
 648 extern PMMPTE MmSharedUserDataPte;
 649 extern LIST_ENTRY MmProcessList;
 650 extern BOOLEAN MmZeroingPageThreadActive;
 651 extern KEVENT MmZeroingPageEvent;
 652 extern ULONG MmSystemPageColor;
 653 extern ULONG MmProcessColorSeed;
 654 extern PMMWSL MmWorkingSetList;
 655 extern PFN_NUMBER MiNumberOfFreePages;
 656 extern SIZE_T MmSessionViewSize;
 657 extern SIZE_T MmSessionPoolSize;
 658 extern SIZE_T MmSessionImageSize;
 659 extern PVOID MiSystemViewStart;
 660 extern PVOID MiSessionPoolEnd;     // 0xBE000000
 661 extern PVOID MiSessionPoolStart;   // 0xBD000000
 662 extern PVOID MiSessionViewStart;   // 0xBE000000
 663 extern PVOID MiSessionSpaceWs;
 664 extern ULONG MmMaximumDeadKernelStacks;
 665 extern SLIST_HEADER MmDeadStackSListHead;
 666 extern MM_AVL_TABLE MmSectionBasedRoot;
 667 extern KGUARDED_MUTEX MmSectionBasedMutex;
 668 extern PVOID MmHighSectionBase;
 669 extern SIZE_T MmSystemLockPagesCount;
 670 extern ULONG_PTR MmSubsectionBase;
 671 extern LARGE_INTEGER MmCriticalSectionTimeout;
 672 extern LIST_ENTRY MmWorkingSetExpansionHead;
 673 extern KSPIN_LOCK MmExpansionLock;
 674 extern PETHREAD MiExpansionLockOwner;
 675
 676 FORCEINLINE
 677 BOOLEAN
 678 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
 679 {
 680     return ((MemoryType == LoaderFree) ||
 681             (MemoryType == LoaderLoadedProgram) ||
 682             (MemoryType == LoaderFirmwareTemporary) ||
 683             (MemoryType == LoaderOsloaderStack));
 684 }
 685
 686 FORCEINLINE
 687 BOOLEAN
 688 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
 689 {
 690     return ((MemoryType == LoaderFirmwarePermanent) ||
 691             (MemoryType == LoaderSpecialMemory) ||
 692             (MemoryType == LoaderHALCachedMemory) ||
 693             (MemoryType == LoaderBBTMemory));
 694 }
 695
 696 #ifdef _M_AMD64
 697 FORCEINLINE
 698 BOOLEAN
 699 MiIsUserPxe(PVOID Address)
 700 {
 701     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
 702 }
 703
 704 FORCEINLINE
 705 BOOLEAN
 706 MiIsUserPpe(PVOID Address)
 707 {
 708     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
 709 }
 710
 711 FORCEINLINE
 712 BOOLEAN
 713 MiIsUserPde(PVOID Address)
 714 {
 715     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
 716 }
 717
 718 FORCEINLINE
 719 BOOLEAN
 720 MiIsUserPte(PVOID Address)
 721 {
 722     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
 723 }
 724 #else
 725 FORCEINLINE
 726 BOOLEAN
 727 MiIsUserPde(PVOID Address)
 728 {
 729     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
 730             (Address <= (PVOID)MiHighestUserPde));
 731 }
 732
 733 FORCEINLINE
 734 BOOLEAN
 735 MiIsUserPte(PVOID Address)
 736 {
 737     return (Address <= (PVOID)MiHighestUserPte);
 738 }
 739 #endif
 740
 741 //
 742 // Figures out the hardware bits for a PTE
 743 //
 744 FORCEINLINE
 745 ULONG_PTR
 746 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
 747 {
 748     MMPTE TempPte;
 749
 750     /* Start fresh */
 751     TempPte.u.Long = 0;
 752
 753     /* Make it valid and accessed */
 754     TempPte.u.Hard.Valid = TRUE;
 755     MI_MAKE_ACCESSED_PAGE(&TempPte);
 756
 757     /* Is this for user-mode? */
 758     if (
 759 #if (_MI_PAGING_LEVELS == 4)
 760         MiIsUserPxe(PointerPte) ||
 761 #endif
 762 #if (_MI_PAGING_LEVELS >= 3)
 763         MiIsUserPpe(PointerPte) ||
 764 #endif
 765         MiIsUserPde(PointerPte) ||
 766         MiIsUserPte(PointerPte))
 767     {
 768         /* Set the owner bit */
 769         MI_MAKE_OWNER_PAGE(&TempPte);
 770     }
 771
 772     /* FIXME: We should also set the global bit */
 773
 774     /* Return the protection */
 775     return TempPte.u.Long;
 776 }
 777
 778 //
 779 // Creates a valid kernel PTE with the given protection
 780 //
 781 FORCEINLINE
 782 VOID
 783 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
 784                             IN PMMPTE MappingPte,
 785                             IN ULONG_PTR ProtectionMask,
 786                             IN PFN_NUMBER PageFrameNumber)
 787 {
 788     /* Only valid for kernel, non-session PTEs */
 789     ASSERT(MappingPte > MiHighestUserPte);
 790     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
 791     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 792
 793     /* Start fresh */
 794     *NewPte = ValidKernelPte;
 795
 796     /* Set the protection and page */
 797     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 798     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 799 }
 800
 801 //
 802 // Creates a valid PTE with the given protection
 803 //
 804 FORCEINLINE
 805 VOID
 806 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
 807                      IN PMMPTE MappingPte,
 808                      IN ULONG_PTR ProtectionMask,
 809                      IN PFN_NUMBER PageFrameNumber)
 810 {
 811     /* Set the protection and page */
 812     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
 813     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 814     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 815 }
 816
 817 //
 818 // Creates a valid user PTE with the given protection
 819 //
 820 FORCEINLINE
 821 VOID
 822 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
 823                           IN PMMPTE MappingPte,
 824                           IN ULONG_PTR ProtectionMask,
 825                           IN PFN_NUMBER PageFrameNumber)
 826 {
 827     /* Only valid for kernel, non-session PTEs */
 828     ASSERT(MappingPte <= MiHighestUserPte);
 829
 830     /* Start fresh */
 831     NewPte->u.Long = 0;
 832
 833     /* Set the protection and page */
 834     NewPte->u.Hard.Valid = TRUE;
 835     NewPte->u.Hard.Owner = TRUE;
 836     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 837     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 838 }
 839
 840 #ifndef _M_AMD64
 841 //
 842 // Builds a Prototype PTE for the address of the PTE
 843 //
 844 FORCEINLINE
 845 VOID
 846 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
 847                       IN PMMPTE PointerPte)
 848 {
 849     ULONG_PTR Offset;
 850
 851     /* Mark this as a prototype */
 852     NewPte->u.Long = 0;
 853     NewPte->u.Proto.Prototype = 1;
 854
 855     /*
 856      * Prototype PTEs are only valid in paged pool by design, this little trick
 857      * lets us only use 30 bits for the adress of the PTE, as long as the area
 858      * stays 1024MB At most.
 859      */
 860     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 861
 862     /*
 863      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
 864      * and the other 21 bits go in the "high"
 865      */
 866     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
 867     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
 868 }
 869
 870 //
 871 // Builds a Subsection PTE for the address of the Segment
 872 //
 873 FORCEINLINE
 874 VOID
 875 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
 876                        IN PVOID Segment)
 877 {
 878     ULONG_PTR Offset;
 879
 880     /* Mark this as a prototype */
 881     NewPte->u.Long = 0;
 882     NewPte->u.Subsect.Prototype = 1;
 883
 884     /*
 885      * Segments are only valid either in nonpaged pool. We store the 20 bit
 886      * difference either from the top or bottom of nonpaged pool, giving a
 887      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
 888      * 256MB.
 889      */
 890     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
 891     {
 892         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
 893         NewPte->u.Subsect.WhichPool = PagedPool;
 894     }
 895     else
 896     {
 897         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
 898         NewPte->u.Subsect.WhichPool = NonPagedPool;
 899     }
 900
 901     /*
 902      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
 903      * and the other 20 bits go in the "high"
 904      */
 905     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
 906     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
 907 }
 908
 909 FORCEINLINE
 910 BOOLEAN
 911 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
 912 {
 913     /// \todo Make this reasonable code, this is UGLY!
 914     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
 915 }
 916
 917 #endif
 918
 919 FORCEINLINE
 920 VOID
 921 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte,
 922                        _In_ PFN_NUMBER Page,
 923                        _In_ ULONG Protection)
 924 {
 925     NewPte->u.Long = 0;
 926     NewPte->u.Trans.Transition = 1;
 927     NewPte->u.Trans.Protection = Protection;
 928     NewPte->u.Trans.PageFrameNumber = Page;
 929 }
 930
 931 //
 932 // Returns if the page is physically resident (ie: a large page)
 933 // FIXFIX: CISC/x86 only?
 934 //
 935 FORCEINLINE
 936 BOOLEAN
 937 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
 938 {
 939     PMMPDE PointerPde;
 940
 941     /* Large pages are never paged out, always physically resident */
 942     PointerPde = MiAddressToPde(Address);
 943     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
 944 }
 945
 946 //
 947 // Writes a valid PTE
 948 //
 949 FORCEINLINE
 950 VOID
 951 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
 952                    IN MMPTE TempPte)
 953 {
 954     /* Write the valid PTE */
 955     ASSERT(PointerPte->u.Hard.Valid == 0);
 956     ASSERT(TempPte.u.Hard.Valid == 1);
 957     *PointerPte = TempPte;
 958 }
 959
 960 //
 961 // Updates a valid PTE
 962 //
 963 FORCEINLINE
 964 VOID
 965 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
 966                    IN MMPTE TempPte)
 967 {
 968     /* Write the valid PTE */
 969     ASSERT(PointerPte->u.Hard.Valid == 1);
 970     ASSERT(TempPte.u.Hard.Valid == 1);
 971     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
 972     *PointerPte = TempPte;
 973 }
 974
 975 //
 976 // Writes an invalid PTE
 977 //
 978 FORCEINLINE
 979 VOID
 980 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
 981                      IN MMPTE InvalidPte)
 982 {
 983     /* Write the invalid PTE */
 984     ASSERT(InvalidPte.u.Hard.Valid == 0);
 985     ASSERT(InvalidPte.u.Long != 0);
 986     *PointerPte = InvalidPte;
 987 }
 988
 989 //
 990 // Erase the PTE completely
 991 //
 992 FORCEINLINE
 993 VOID
 994 MI_ERASE_PTE(IN PMMPTE PointerPte)
 995 {
 996     /* Zero out the PTE */
 997     ASSERT(PointerPte->u.Long != 0);
 998     PointerPte->u.Long = 0;
 999 }
1000
1001 //
1002 // Writes a valid PDE
1003 //
1004 FORCEINLINE
1005 VOID
1006 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1007                    IN MMPDE TempPde)
1008 {
1009     /* Write the valid PDE */
1010     ASSERT(PointerPde->u.Hard.Valid == 0);
1011     ASSERT(TempPde.u.Hard.Valid == 1);
1012     *PointerPde = TempPde;
1013 }
1014
1015 //
1016 // Writes an invalid PDE
1017 //
1018 FORCEINLINE
1019 VOID
1020 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1021                      IN MMPDE InvalidPde)
1022 {
1023     /* Write the invalid PDE */
1024     ASSERT(InvalidPde.u.Hard.Valid == 0);
1025     ASSERT(InvalidPde.u.Long != 0);
1026     *PointerPde = InvalidPde;
1027 }
1028
1029 //
1030 // Checks if the thread already owns a working set
1031 //
1032 FORCEINLINE
1033 BOOLEAN
1034 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1035 {
1036     /* If any of these are held, return TRUE */
1037     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1038             (Thread->OwnsProcessWorkingSetShared) ||
1039             (Thread->OwnsSystemWorkingSetExclusive) ||
1040             (Thread->OwnsSystemWorkingSetShared) ||
1041             (Thread->OwnsSessionWorkingSetExclusive) ||
1042             (Thread->OwnsSessionWorkingSetShared));
1043 }
1044
1045 //
1046 // Checks if the process owns the working set lock
1047 //
1048 FORCEINLINE
1049 BOOLEAN
1050 MI_WS_OWNER(IN PEPROCESS Process)
1051 {
1052     /* Check if this process is the owner, and that the thread owns the WS */
1053     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1054     {
1055         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1056     }
1057     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1058     {
1059         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1060     }
1061     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1062             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1063              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1064 }
1065
1066 //
1067 // New ARM3<->RosMM PAGE Architecture
1068 //
1069 FORCEINLINE
1070 BOOLEAN
1071 MiIsRosSectionObject(IN PVOID Section)
1072 {
1073     PROS_SECTION_OBJECT RosSection = Section;
1074     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1075     return FALSE;
1076 }
1077
1078 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1079
1080 VOID
1081 NTAPI
1082 MiDecrementReferenceCount(
1083     IN PMMPFN Pfn1,
1084     IN PFN_NUMBER PageFrameIndex
1085 );
1086
1087 FORCEINLINE
1088 BOOLEAN
1089 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1090 {
1091     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1092 }
1093
1094 //
1095 // Locks the working set for the given process
1096 //
1097 FORCEINLINE
1098 VOID
1099 MiLockProcessWorkingSet(IN PEPROCESS Process,
1100                         IN PETHREAD Thread)
1101 {
1102     /* Shouldn't already be owning the process working set */
1103     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1104     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1105
1106     /* Block APCs, make sure that still nothing is already held */
1107     KeEnterGuardedRegion();
1108     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1109
1110     /* Lock the working set */
1111     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1112
1113     /* Now claim that we own the lock */
1114     ASSERT(!MI_IS_WS_UNSAFE(Process));
1115     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1116     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1117 }
1118
1119 FORCEINLINE
1120 VOID
1121 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1122                               IN PETHREAD Thread)
1123 {
1124     /* Shouldn't already be owning the process working set */
1125     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1126     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1127
1128     /* Block APCs, make sure that still nothing is already held */
1129     KeEnterGuardedRegion();
1130     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1131
1132     /* Lock the working set */
1133     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1134
1135     /* Now claim that we own the lock */
1136     ASSERT(!MI_IS_WS_UNSAFE(Process));
1137     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1138     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1139     Thread->OwnsProcessWorkingSetShared = TRUE;
1140 }
1141
1142 FORCEINLINE
1143 VOID
1144 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1145                               IN PETHREAD Thread)
1146 {
1147     /* Shouldn't already be owning the process working set */
1148     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1149
1150     /* APCs must be blocked, make sure that still nothing is already held */
1151     ASSERT(KeAreAllApcsDisabled() == TRUE);
1152     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1153
1154     /* Lock the working set */
1155     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1156
1157     /* Now claim that we own the lock */
1158     ASSERT(!MI_IS_WS_UNSAFE(Process));
1159     Process->Vm.Flags.AcquiredUnsafe = 1;
1160     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1161     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1162 }
1163
1164 //
1165 // Unlocks the working set for the given process
1166 //
1167 FORCEINLINE
1168 VOID
1169 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1170                           IN PETHREAD Thread)
1171 {
1172     /* Make sure we are the owner of a safe acquisition */
1173     ASSERT(MI_WS_OWNER(Process));
1174     ASSERT(!MI_IS_WS_UNSAFE(Process));
1175
1176     /* The thread doesn't own it anymore */
1177     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1178     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1179
1180     /* Release the lock and re-enable APCs */
1181     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1182     KeLeaveGuardedRegion();
1183 }
1184
1185 //
1186 // Unlocks the working set for the given process
1187 //
1188 FORCEINLINE
1189 VOID
1190 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1191                                 IN PETHREAD Thread)
1192 {
1193     /* Make sure we are the owner of a safe acquisition (because shared) */
1194     ASSERT(MI_WS_OWNER(Process));
1195     ASSERT(!MI_IS_WS_UNSAFE(Process));
1196
1197     /* Ensure we are in a shared acquisition */
1198     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1199     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1200
1201     /* Don't claim the lock anylonger */
1202     Thread->OwnsProcessWorkingSetShared = FALSE;
1203
1204     /* Release the lock and re-enable APCs */
1205     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1206     KeLeaveGuardedRegion();
1207 }
1208
1209 //
1210 // Unlocks the working set for the given process
1211 //
1212 FORCEINLINE
1213 VOID
1214 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1215                                 IN PETHREAD Thread)
1216 {
1217     /* Make sure we are the owner of an unsafe acquisition */
1218     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1219     ASSERT(KeAreAllApcsDisabled() == TRUE);
1220     ASSERT(MI_WS_OWNER(Process));
1221     ASSERT(MI_IS_WS_UNSAFE(Process));
1222
1223     /* No longer unsafe */
1224     Process->Vm.Flags.AcquiredUnsafe = 0;
1225
1226     /* The thread doesn't own it anymore */
1227     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1228     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1229
1230     /* Release the lock but don't touch APC state */
1231     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1232     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1233 }
1234
1235 //
1236 // Locks the working set
1237 //
1238 FORCEINLINE
1239 VOID
1240 MiLockWorkingSet(IN PETHREAD Thread,
1241                  IN PMMSUPPORT WorkingSet)
1242 {
1243     /* Block APCs */
1244     KeEnterGuardedRegion();
1245
1246     /* Working set should be in global memory */
1247     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1248
1249     /* Thread shouldn't already be owning something */
1250     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1251
1252     /* Lock this working set */
1253     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1254
1255     /* Which working set is this? */
1256     if (WorkingSet == &MmSystemCacheWs)
1257     {
1258         /* Own the system working set */
1259         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1260                (Thread->OwnsSystemWorkingSetShared == FALSE));
1261         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1262     }
1263     else if (WorkingSet->Flags.SessionSpace)
1264     {
1265         /* Own the session working set */
1266         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1267                (Thread->OwnsSessionWorkingSetShared == FALSE));
1268         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1269     }
1270     else
1271     {
1272         /* Own the process working set */
1273         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1274                (Thread->OwnsProcessWorkingSetShared == FALSE));
1275         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1276     }
1277 }
1278
1279 //
1280 // Unlocks the working set
1281 //
1282 FORCEINLINE
1283 VOID
1284 MiUnlockWorkingSet(IN PETHREAD Thread,
1285                    IN PMMSUPPORT WorkingSet)
1286 {
1287     /* Working set should be in global memory */
1288     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1289
1290     /* Which working set is this? */
1291     if (WorkingSet == &MmSystemCacheWs)
1292     {
1293         /* Release the system working set */
1294         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1295                (Thread->OwnsSystemWorkingSetShared == TRUE));
1296         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1297     }
1298     else if (WorkingSet->Flags.SessionSpace)
1299     {
1300         /* Release the session working set */
1301         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1302                (Thread->OwnsSessionWorkingSetShared == TRUE));
1303         Thread->OwnsSessionWorkingSetExclusive = 0;
1304     }
1305     else
1306     {
1307         /* Release the process working set */
1308         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1309                (Thread->OwnsProcessWorkingSetShared));
1310         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1311     }
1312
1313     /* Release the working set lock */
1314     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1315
1316     /* Unblock APCs */
1317     KeLeaveGuardedRegion();
1318 }
1319
1320 FORCEINLINE
1321 VOID
1322 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1323                                   IN PETHREAD Thread,
1324                                   OUT PBOOLEAN Safe,
1325                                   OUT PBOOLEAN Shared)
1326 {
1327     ASSERT(MI_WS_OWNER(Process));
1328
1329     /* Check if the current owner is unsafe */
1330     if (MI_IS_WS_UNSAFE(Process))
1331     {
1332         /* Release unsafely */
1333         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1334         *Safe = FALSE;
1335         *Shared = FALSE;
1336     }
1337     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1338     {
1339         /* Owner is safe and exclusive, release normally */
1340         MiUnlockProcessWorkingSet(Process, Thread);
1341         *Safe = TRUE;
1342         *Shared = FALSE;
1343     }
1344     else
1345     {
1346         /* Owner is shared (implies safe), release normally */
1347         MiUnlockProcessWorkingSetShared(Process, Thread);
1348         *Safe = TRUE;
1349         *Shared = TRUE;
1350     }
1351 }
1352
1353 FORCEINLINE
1354 VOID
1355 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1356                                 IN PETHREAD Thread,
1357                                 IN BOOLEAN Safe,
1358                                 IN BOOLEAN Shared)
1359 {
1360     /* Check if this was a safe lock or not */
1361     if (Safe)
1362     {
1363         if (Shared)
1364         {
1365             /* Reacquire safely & shared */
1366             MiLockProcessWorkingSetShared(Process, Thread);
1367         }
1368         else
1369         {
1370             /* Reacquire safely */
1371             MiLockProcessWorkingSet(Process, Thread);
1372         }
1373     }
1374     else
1375     {
1376         /* Unsafe lock cannot be shared */
1377         ASSERT(Shared == FALSE);
1378         /* Reacquire unsafely */
1379         MiLockProcessWorkingSetUnsafe(Process, Thread);
1380     }
1381 }
1382
1383 FORCEINLINE
1384 KIRQL
1385 MiAcquireExpansionLock(VOID)
1386 {
1387     KIRQL OldIrql;
1388
1389     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1390     KeAcquireSpinLock(&MmExpansionLock, &OldIrql);
1391     ASSERT(MiExpansionLockOwner == NULL);
1392     MiExpansionLockOwner = PsGetCurrentThread();
1393     return OldIrql;
1394 }
1395
1396 FORCEINLINE
1397 VOID
1398 MiReleaseExpansionLock(KIRQL OldIrql)
1399 {
1400     ASSERT(MiExpansionLockOwner == PsGetCurrentThread());
1401     MiExpansionLockOwner = NULL;
1402     KeReleaseSpinLock(&MmExpansionLock, OldIrql);
1403     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1404 }
1405
1406 //
1407 // Returns the ProtoPTE inside a VAD for the given VPN
1408 //
1409 FORCEINLINE
1410 PMMPTE
1411 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1412                              IN ULONG_PTR Vpn)
1413 {
1414     PMMPTE ProtoPte;
1415
1416     /* Find the offset within the VAD's prototype PTEs */
1417     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1418     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1419     return ProtoPte;
1420 }
1421
1422 //
1423 // Returns the PFN Database entry for the given page number
1424 // Warning: This is not necessarily a valid PFN database entry!
1425 //
1426 FORCEINLINE
1427 PMMPFN
1428 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1429 {
1430     /* Get the entry */
1431     return &MmPfnDatabase[Pfn];
1432 };
1433
1434 //
1435 // Drops a locked page without dereferencing it
1436 //
1437 FORCEINLINE
1438 VOID
1439 MiDropLockCount(IN PMMPFN Pfn1)
1440 {
1441     /* This page shouldn't be locked, but it should be valid */
1442     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1443     ASSERT(Pfn1->u2.ShareCount == 0);
1444
1445     /* Is this the last reference to the page */
1446     if (Pfn1->u3.e2.ReferenceCount == 1)
1447     {
1448         /* It better not be valid */
1449         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1450
1451         /* Is it a prototype PTE? */
1452         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1453             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1454         {
1455             /* FIXME: We should return commit */
1456             DPRINT1("Not returning commit for prototype PTE\n");
1457         }
1458
1459         /* Update the counter */
1460         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1461     }
1462 }
1463
1464 //
1465 // Drops a locked page and dereferences it
1466 //
1467 FORCEINLINE
1468 VOID
1469 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1470 {
1471     USHORT RefCount, OldRefCount;
1472     PFN_NUMBER PageFrameIndex;
1473
1474     /* Loop while we decrement the page successfully */
1475     do
1476     {
1477         /* There should be at least one reference */
1478         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1479         ASSERT(OldRefCount != 0);
1480
1481         /* Are we the last one */
1482         if (OldRefCount == 1)
1483         {
1484             /* The page shoudln't be shared not active at this point */
1485             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1486             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1487             ASSERT(Pfn1->u2.ShareCount == 0);
1488
1489             /* Is it a prototype PTE? */
1490             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1491                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1492             {
1493                 /* FIXME: We should return commit */
1494                 DPRINT1("Not returning commit for prototype PTE\n");
1495             }
1496
1497             /* Update the counter, and drop a reference the long way */
1498             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1499             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1500             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1501             return;
1502         }
1503
1504         /* Drop a reference the short way, and that's it */
1505         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1506                                                 OldRefCount - 1,
1507                                                 OldRefCount);
1508         ASSERT(RefCount != 0);
1509     } while (OldRefCount != RefCount);
1510
1511     /* If we got here, there should be more than one reference */
1512     ASSERT(RefCount > 1);
1513     if (RefCount == 2)
1514     {
1515         /* Is it still being shared? */
1516         if (Pfn1->u2.ShareCount >= 1)
1517         {
1518             /* Then it should be valid */
1519             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1520
1521             /* Is it a prototype PTE? */
1522             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1523                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1524             {
1525                 /* We don't handle ethis */
1526                 ASSERT(FALSE);
1527             }
1528
1529             /* Update the counter */
1530             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1531         }
1532     }
1533 }
1534
1535 //
1536 // References a locked page and updates the counter
1537 // Used in MmProbeAndLockPages to handle different edge cases
1538 //
1539 FORCEINLINE
1540 VOID
1541 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1542 {
1543     USHORT RefCount, OldRefCount;
1544
1545     /* Sanity check */
1546     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1547
1548     /* Does ARM3 own the page? */
1549     if (MI_IS_ROS_PFN(Pfn1))
1550     {
1551         /* ReactOS Mm doesn't track share count */
1552         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1553     }
1554     else
1555     {
1556         /* On ARM3 pages, we should see a valid share count */
1557         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1558
1559         /* Is it a prototype PTE? */
1560         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1561             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1562         {
1563             /* FIXME: We should charge commit */
1564             DPRINT1("Not charging commit for prototype PTE\n");
1565         }
1566     }
1567
1568     /* More locked pages! */
1569     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1570
1571     /* Loop trying to update the reference count */
1572     do
1573     {
1574         /* Get the current reference count, make sure it's valid */
1575         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1576         ASSERT(OldRefCount != 0);
1577         ASSERT(OldRefCount < 2500);
1578
1579         /* Bump it up by one */
1580         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1581                                                 OldRefCount + 1,
1582                                                 OldRefCount);
1583         ASSERT(RefCount != 0);
1584     } while (OldRefCount != RefCount);
1585
1586     /* Was this the first lock attempt? If not, undo our bump */
1587     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1588 }
1589
1590 //
1591 // References a locked page and updates the counter
1592 // Used in all other cases except MmProbeAndLockPages
1593 //
1594 FORCEINLINE
1595 VOID
1596 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1597 {
1598     USHORT NewRefCount;
1599
1600     /* Is it a prototype PTE? */
1601     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1602         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1603     {
1604         /* FIXME: We should charge commit */
1605         DPRINT1("Not charging commit for prototype PTE\n");
1606     }
1607
1608     /* More locked pages! */
1609     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1610
1611     /* Update the reference count */
1612     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1613     if (NewRefCount == 2)
1614     {
1615         /* Is it locked or shared? */
1616         if (Pfn1->u2.ShareCount)
1617         {
1618             /* It's shared, so make sure it's active */
1619             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1620         }
1621         else
1622         {
1623             /* It's locked, so we shouldn't lock again */
1624             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1625         }
1626     }
1627     else
1628     {
1629         /* Someone had already locked the page, so undo our bump */
1630         ASSERT(NewRefCount < 2500);
1631         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1632     }
1633 }
1634
1635 //
1636 // References a locked page and updates the counter
1637 // Used in all other cases except MmProbeAndLockPages
1638 //
1639 FORCEINLINE
1640 VOID
1641 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1642 {
1643     USHORT NewRefCount;
1644
1645     /* Make sure the page isn't used yet */
1646     ASSERT(Pfn1->u2.ShareCount == 0);
1647     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1648
1649     /* Is it a prototype PTE? */
1650     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1651         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1652     {
1653         /* FIXME: We should charge commit */
1654         DPRINT1("Not charging commit for prototype PTE\n");
1655     }
1656
1657     /* More locked pages! */
1658     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1659
1660     /* Update the reference count */
1661     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1662     if (NewRefCount != 1)
1663     {
1664         /* Someone had already locked the page, so undo our bump */
1665         ASSERT(NewRefCount < 2500);
1666         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1667     }
1668 }
1669
1670 FORCEINLINE
1671 VOID
1672 MiIncrementPageTableReferences(IN PVOID Address)
1673 {
1674     PUSHORT RefCount;
1675
1676     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1677
1678     *RefCount += 1;
1679     ASSERT(*RefCount <= PTE_PER_PAGE);
1680 }
1681
1682 FORCEINLINE
1683 VOID
1684 MiDecrementPageTableReferences(IN PVOID Address)
1685 {
1686     PUSHORT RefCount;
1687
1688     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1689
1690     *RefCount -= 1;
1691     ASSERT(*RefCount < PTE_PER_PAGE);
1692 }
1693
1694 FORCEINLINE
1695 USHORT
1696 MiQueryPageTableReferences(IN PVOID Address)
1697 {
1698     PUSHORT RefCount;
1699
1700     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1701
1702     return *RefCount;
1703 }
1704
1705 INIT_FUNCTION
1706 BOOLEAN
1707 NTAPI
1708 MmArmInitSystem(
1709     IN ULONG Phase,
1710     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1711 );
1712
1713 INIT_FUNCTION
1714 VOID
1715 NTAPI
1716 MiInitializeSessionSpaceLayout(VOID);
1717
1718 INIT_FUNCTION
1719 NTSTATUS
1720 NTAPI
1721 MiInitMachineDependent(
1722     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1723 );
1724
1725 INIT_FUNCTION
1726 VOID
1727 NTAPI
1728 MiComputeColorInformation(
1729     VOID
1730 );
1731
1732 INIT_FUNCTION
1733 VOID
1734 NTAPI
1735 MiMapPfnDatabase(
1736     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1737 );
1738
1739 INIT_FUNCTION
1740 VOID
1741 NTAPI
1742 MiInitializeColorTables(
1743     VOID
1744 );
1745
1746 INIT_FUNCTION
1747 VOID
1748 NTAPI
1749 MiInitializePfnDatabase(
1750     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1751 );
1752
1753 VOID
1754 NTAPI
1755 MiInitializeSessionWsSupport(
1756     VOID
1757 );
1758
1759 VOID
1760 NTAPI
1761 MiInitializeSessionIds(
1762     VOID
1763 );
1764
1765 INIT_FUNCTION
1766 BOOLEAN
1767 NTAPI
1768 MiInitializeMemoryEvents(
1769     VOID
1770 );
1771
1772 INIT_FUNCTION
1773 PFN_NUMBER
1774 NTAPI
1775 MxGetNextPage(
1776     IN PFN_NUMBER PageCount
1777 );
1778
1779 INIT_FUNCTION
1780 PPHYSICAL_MEMORY_DESCRIPTOR
1781 NTAPI
1782 MmInitializeMemoryLimits(
1783     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1784     IN PBOOLEAN IncludeType
1785 );
1786
1787 PFN_NUMBER
1788 NTAPI
1789 MiPagesInLoaderBlock(
1790     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1791     IN PBOOLEAN IncludeType
1792 );
1793
1794 VOID
1795 FASTCALL
1796 MiSyncARM3WithROS(
1797     IN PVOID AddressStart,
1798     IN PVOID AddressEnd
1799 );
1800
1801 NTSTATUS
1802 NTAPI
1803 MiRosProtectVirtualMemory(
1804     IN PEPROCESS Process,
1805     IN OUT PVOID *BaseAddress,
1806     IN OUT PSIZE_T NumberOfBytesToProtect,
1807     IN ULONG NewAccessProtection,
1808     OUT PULONG OldAccessProtection OPTIONAL
1809 );
1810
1811 NTSTATUS
1812 NTAPI
1813 MmArmAccessFault(
1814     IN ULONG FaultCode,
1815     IN PVOID Address,
1816     IN KPROCESSOR_MODE Mode,
1817     IN PVOID TrapInformation
1818 );
1819
1820 NTSTATUS
1821 FASTCALL
1822 MiCheckPdeForPagedPool(
1823     IN PVOID Address
1824 );
1825
1826 INIT_FUNCTION
1827 VOID
1828 NTAPI
1829 MiInitializeNonPagedPool(
1830     VOID
1831 );
1832
1833 INIT_FUNCTION
1834 VOID
1835 NTAPI
1836 MiInitializeNonPagedPoolThresholds(
1837     VOID
1838 );
1839
1840 INIT_FUNCTION
1841 VOID
1842 NTAPI
1843 MiInitializePoolEvents(
1844     VOID
1845 );
1846
1847 INIT_FUNCTION
1848 VOID                      //
1849 NTAPI                     //
1850 InitializePool(           //
1851     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1852     IN ULONG Threshold    //
1853 );                        //
1854
1855 // FIXFIX: THIS ONE TOO
1856 INIT_FUNCTION
1857 VOID
1858 NTAPI
1859 ExInitializePoolDescriptor(
1860     IN PPOOL_DESCRIPTOR PoolDescriptor,
1861     IN POOL_TYPE PoolType,
1862     IN ULONG PoolIndex,
1863     IN ULONG Threshold,
1864     IN PVOID PoolLock
1865 );
1866
1867 NTSTATUS
1868 NTAPI
1869 MiInitializeSessionPool(
1870     VOID
1871 );
1872
1873 INIT_FUNCTION
1874 VOID
1875 NTAPI
1876 MiInitializeSystemPtes(
1877     IN PMMPTE StartingPte,
1878     IN ULONG NumberOfPtes,
1879     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1880 );
1881
1882 PMMPTE
1883 NTAPI
1884 MiReserveSystemPtes(
1885     IN ULONG NumberOfPtes,
1886     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1887 );
1888
1889 VOID
1890 NTAPI
1891 MiReleaseSystemPtes(
1892     IN PMMPTE StartingPte,
1893     IN ULONG NumberOfPtes,
1894     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1895 );
1896
1897
1898 PFN_NUMBER
1899 NTAPI
1900 MiFindContiguousPages(
1901     IN PFN_NUMBER LowestPfn,
1902     IN PFN_NUMBER HighestPfn,
1903     IN PFN_NUMBER BoundaryPfn,
1904     IN PFN_NUMBER SizeInPages,
1905     IN MEMORY_CACHING_TYPE CacheType
1906 );
1907
1908 PVOID
1909 NTAPI
1910 MiCheckForContiguousMemory(
1911     IN PVOID BaseAddress,
1912     IN PFN_NUMBER BaseAddressPages,
1913     IN PFN_NUMBER SizeInPages,
1914     IN PFN_NUMBER LowestPfn,
1915     IN PFN_NUMBER HighestPfn,
1916     IN PFN_NUMBER BoundaryPfn,
1917     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1918 );
1919
1920 PMDL
1921 NTAPI
1922 MiAllocatePagesForMdl(
1923     IN PHYSICAL_ADDRESS LowAddress,
1924     IN PHYSICAL_ADDRESS HighAddress,
1925     IN PHYSICAL_ADDRESS SkipBytes,
1926     IN SIZE_T TotalBytes,
1927     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1928     IN ULONG Flags
1929 );
1930
1931 VOID
1932 NTAPI
1933 MiInsertPageInList(
1934     IN PMMPFNLIST ListHead,
1935     IN PFN_NUMBER PageFrameIndex
1936 );
1937
1938 VOID
1939 NTAPI
1940 MiUnlinkFreeOrZeroedPage(
1941     IN PMMPFN Entry
1942 );
1943
1944 VOID
1945 NTAPI
1946 MiUnlinkPageFromList(
1947     IN PMMPFN Pfn
1948 );
1949
1950 VOID
1951 NTAPI
1952 MiInitializePfn(
1953     IN PFN_NUMBER PageFrameIndex,
1954     IN PMMPTE PointerPte,
1955     IN BOOLEAN Modified
1956 );
1957
1958 NTSTATUS
1959 NTAPI
1960 MiInitializeAndChargePfn(
1961     OUT PPFN_NUMBER PageFrameIndex,
1962     IN PMMPDE PointerPde,
1963     IN PFN_NUMBER ContainingPageFrame,
1964     IN BOOLEAN SessionAllocation
1965 );
1966
1967 VOID
1968 NTAPI
1969 MiInitializePfnAndMakePteValid(
1970     IN PFN_NUMBER PageFrameIndex,
1971     IN PMMPTE PointerPte,
1972     IN MMPTE TempPte
1973 );
1974
1975 VOID
1976 NTAPI
1977 MiInitializePfnForOtherProcess(
1978     IN PFN_NUMBER PageFrameIndex,
1979     IN PVOID PteAddress,
1980     IN PFN_NUMBER PteFrame
1981 );
1982
1983 VOID
1984 NTAPI
1985 MiDecrementShareCount(
1986     IN PMMPFN Pfn1,
1987     IN PFN_NUMBER PageFrameIndex
1988 );
1989
1990 PFN_NUMBER
1991 NTAPI
1992 MiRemoveAnyPage(
1993     IN ULONG Color
1994 );
1995
1996 PFN_NUMBER
1997 NTAPI
1998 MiRemoveZeroPage(
1999     IN ULONG Color
2000 );
2001
2002 VOID
2003 NTAPI
2004 MiZeroPhysicalPage(
2005     IN PFN_NUMBER PageFrameIndex
2006 );
2007
2008 VOID
2009 NTAPI
2010 MiInsertPageInFreeList(
2011     IN PFN_NUMBER PageFrameIndex
2012 );
2013
2014 PFN_COUNT
2015 NTAPI
2016 MiDeleteSystemPageableVm(
2017     IN PMMPTE PointerPte,
2018     IN PFN_NUMBER PageCount,
2019     IN ULONG Flags,
2020     OUT PPFN_NUMBER ValidPages
2021 );
2022
2023 ULONG
2024 NTAPI
2025 MiGetPageProtection(
2026     IN PMMPTE PointerPte
2027 );
2028
2029 PLDR_DATA_TABLE_ENTRY
2030 NTAPI
2031 MiLookupDataTableEntry(
2032     IN PVOID Address
2033 );
2034
2035 INIT_FUNCTION
2036 VOID
2037 NTAPI
2038 MiInitializeDriverLargePageList(
2039     VOID
2040 );
2041
2042 INIT_FUNCTION
2043 VOID
2044 NTAPI
2045 MiInitializeLargePageSupport(
2046     VOID
2047 );
2048
2049 INIT_FUNCTION
2050 VOID
2051 NTAPI
2052 MiSyncCachedRanges(
2053     VOID
2054 );
2055
2056 BOOLEAN
2057 NTAPI
2058 MiIsPfnInUse(
2059     IN PMMPFN Pfn1
2060 );
2061
2062 PMMVAD
2063 NTAPI
2064 MiLocateAddress(
2065     IN PVOID VirtualAddress
2066 );
2067
2068 TABLE_SEARCH_RESULT
2069 NTAPI
2070 MiCheckForConflictingNode(
2071     IN ULONG_PTR StartVpn,
2072     IN ULONG_PTR EndVpn,
2073     IN PMM_AVL_TABLE Table,
2074     OUT PMMADDRESS_NODE *NodeOrParent
2075 );
2076
2077 TABLE_SEARCH_RESULT
2078 NTAPI
2079 MiFindEmptyAddressRangeDownTree(
2080     IN SIZE_T Length,
2081     IN ULONG_PTR BoundaryAddress,
2082     IN ULONG_PTR Alignment,
2083     IN PMM_AVL_TABLE Table,
2084     OUT PULONG_PTR Base,
2085     OUT PMMADDRESS_NODE *Parent
2086 );
2087
2088 NTSTATUS
2089 NTAPI
2090 MiFindEmptyAddressRangeDownBasedTree(
2091     IN SIZE_T Length,
2092     IN ULONG_PTR BoundaryAddress,
2093     IN ULONG_PTR Alignment,
2094     IN PMM_AVL_TABLE Table,
2095     OUT PULONG_PTR Base
2096 );
2097
2098 TABLE_SEARCH_RESULT
2099 NTAPI
2100 MiFindEmptyAddressRangeInTree(
2101     IN SIZE_T Length,
2102     IN ULONG_PTR Alignment,
2103     IN PMM_AVL_TABLE Table,
2104     OUT PMMADDRESS_NODE *PreviousVad,
2105     OUT PULONG_PTR Base
2106 );
2107
2108 NTSTATUS
2109 NTAPI
2110 MiCheckSecuredVad(
2111     IN PMMVAD Vad,
2112     IN PVOID Base,
2113     IN SIZE_T Size,
2114     IN ULONG ProtectionMask
2115 );
2116
2117 VOID
2118 NTAPI
2119 MiInsertVad(
2120     _Inout_ PMMVAD Vad,
2121     _Inout_ PMM_AVL_TABLE VadRoot);
2122
2123 NTSTATUS
2124 NTAPI
2125 MiInsertVadEx(
2126     _Inout_ PMMVAD Vad,
2127     _In_ ULONG_PTR *BaseAddress,
2128     _In_ SIZE_T ViewSize,
2129     _In_ ULONG_PTR HighestAddress,
2130     _In_ ULONG_PTR Alignment,
2131     _In_ ULONG AllocationType);
2132
2133 VOID
2134 NTAPI
2135 MiInsertBasedSection(
2136     IN PSECTION Section
2137 );
2138
2139 NTSTATUS
2140 NTAPI
2141 MiUnmapViewOfSection(
2142     IN PEPROCESS Process,
2143     IN PVOID BaseAddress,
2144     IN ULONG Flags
2145 );
2146
2147 NTSTATUS
2148 NTAPI
2149 MiRosUnmapViewOfSection(
2150     IN PEPROCESS Process,
2151     IN PVOID BaseAddress,
2152     IN BOOLEAN SkipDebuggerNotify
2153 );
2154
2155 VOID
2156 NTAPI
2157 MiInsertNode(
2158     IN PMM_AVL_TABLE Table,
2159     IN PMMADDRESS_NODE NewNode,
2160     PMMADDRESS_NODE Parent,
2161     TABLE_SEARCH_RESULT Result
2162 );
2163
2164 VOID
2165 NTAPI
2166 MiRemoveNode(
2167     IN PMMADDRESS_NODE Node,
2168     IN PMM_AVL_TABLE Table
2169 );
2170
2171 PMMADDRESS_NODE
2172 NTAPI
2173 MiGetPreviousNode(
2174     IN PMMADDRESS_NODE Node
2175 );
2176
2177 PMMADDRESS_NODE
2178 NTAPI
2179 MiGetNextNode(
2180     IN PMMADDRESS_NODE Node
2181 );
2182
2183 BOOLEAN
2184 NTAPI
2185 MiInitializeSystemSpaceMap(
2186     IN PMMSESSION InputSession OPTIONAL
2187 );
2188
2189 VOID
2190 NTAPI
2191 MiSessionRemoveProcess(
2192     VOID
2193 );
2194
2195 VOID
2196 NTAPI
2197 MiReleaseProcessReferenceToSessionDataPage(
2198     IN PMM_SESSION_SPACE SessionGlobal
2199 );
2200
2201 VOID
2202 NTAPI
2203 MiSessionAddProcess(
2204     IN PEPROCESS NewProcess
2205 );
2206
2207 NTSTATUS
2208 NTAPI
2209 MiSessionCommitPageTables(
2210     IN PVOID StartVa,
2211     IN PVOID EndVa
2212 );
2213
2214 ULONG
2215 NTAPI
2216 MiMakeProtectionMask(
2217     IN ULONG Protect
2218 );
2219
2220 VOID
2221 NTAPI
2222 MiDeleteVirtualAddresses(
2223     IN ULONG_PTR Va,
2224     IN ULONG_PTR EndingAddress,
2225     IN PMMVAD Vad
2226 );
2227
2228 VOID
2229 NTAPI
2230 MiDeletePte(
2231     IN PMMPTE PointerPte,
2232     IN PVOID VirtualAddress,
2233     IN PEPROCESS CurrentProcess,
2234     IN PMMPTE PrototypePte
2235 );
2236
2237 ULONG
2238 NTAPI
2239 MiMakeSystemAddressValid(
2240     IN PVOID PageTableVirtualAddress,
2241     IN PEPROCESS CurrentProcess
2242 );
2243
2244 ULONG
2245 NTAPI
2246 MiMakeSystemAddressValidPfn(
2247     IN PVOID VirtualAddress,
2248     IN KIRQL OldIrql
2249 );
2250
2251 VOID
2252 NTAPI
2253 MiRemoveMappedView(
2254     IN PEPROCESS CurrentProcess,
2255     IN PMMVAD Vad
2256 );
2257
2258 PSUBSECTION
2259 NTAPI
2260 MiLocateSubsection(
2261     IN PMMVAD Vad,
2262     IN ULONG_PTR Vpn
2263 );
2264
2265 VOID
2266 NTAPI
2267 MiDeleteARM3Section(
2268     PVOID ObjectBody
2269 );
2270
2271 NTSTATUS
2272 NTAPI
2273 MiQueryMemorySectionName(
2274     IN HANDLE ProcessHandle,
2275     IN PVOID BaseAddress,
2276     OUT PVOID MemoryInformation,
2277     IN SIZE_T MemoryInformationLength,
2278     OUT PSIZE_T ReturnLength
2279 );
2280
2281 NTSTATUS
2282 NTAPI
2283 MiRosUnmapViewInSystemSpace(
2284     IN PVOID MappedBase
2285 );
2286
2287 POOL_TYPE
2288 NTAPI
2289 MmDeterminePoolType(
2290     IN PVOID PoolAddress
2291 );
2292
2293 VOID
2294 NTAPI
2295 MiMakePdeExistAndMakeValid(
2296     IN PMMPDE PointerPde,
2297     IN PEPROCESS TargetProcess,
2298     IN KIRQL OldIrql
2299 );
2300
2301 VOID
2302 NTAPI
2303 MiWriteProtectSystemImage(
2304     _In_ PVOID ImageBase);
2305
2306 //
2307 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2308 // free pages are available. In some scenarios, we don't/can't run that piece of
2309 // code and would rather only have a real zero page. If we can't have a zero page,
2310 // then we'd like to have our own code to grab a free page and zero it out, by
2311 // using MiRemoveAnyPage. This macro implements this.
2312 //
2313 FORCEINLINE
2314 PFN_NUMBER
2315 MiRemoveZeroPageSafe(IN ULONG Color)
2316 {
2317     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2318     return 0;
2319 }
2320
2321 #if (_MI_PAGING_LEVELS == 2)
2322 FORCEINLINE
2323 BOOLEAN
2324 MiSynchronizeSystemPde(PMMPDE PointerPde)
2325 {
2326     MMPDE SystemPde;
2327     ULONG Index;
2328
2329     /* Get the Index from the PDE */
2330     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
2331
2332     /* Copy the PDE from the double-mapped system page directory */
2333     SystemPde = MmSystemPagePtes[Index];
2334     *PointerPde = SystemPde;
2335
2336     /* Make sure we re-read the PDE and PTE */
2337     KeMemoryBarrierWithoutFence();
2338
2339     /* Return, if we had success */
2340     return SystemPde.u.Hard.Valid != 0;
2341 }
2342 #endif
2343
2344 /* EOF */