ntoskrnl/mm/ARM3/miarm.h

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
   5  * PURPOSE:         ARM Memory Manager Header
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 #ifndef _M_AMD64
  10
  11 #define MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING   ((255 * _1MB) >> PAGE_SHIFT)
  12 #define MI_MIN_PAGES_FOR_SYSPTE_TUNING          ((19 * _1MB) >> PAGE_SHIFT)
  13 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST           ((32 * _1MB) >> PAGE_SHIFT)
  14 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST_BOOST     ((256 * _1MB) >> PAGE_SHIFT)
  15 #define MI_MAX_INIT_NONPAGED_POOL_SIZE          (128 * _1MB)
  16 #define MI_MAX_NONPAGED_POOL_SIZE               (128 * _1MB)
  17 #define MI_MAX_FREE_PAGE_LISTS                  4
  18
  19 #define MI_MIN_INIT_PAGED_POOLSIZE              (32 * _1MB)
  20
  21 #define MI_SESSION_VIEW_SIZE                    (20 * _1MB)
  22 #define MI_SESSION_POOL_SIZE                    (16 * _1MB)
  23 #define MI_SESSION_IMAGE_SIZE                   (8 * _1MB)
  24 #define MI_SESSION_WORKING_SET_SIZE             (4 * _1MB)
  25 #define MI_SESSION_SIZE                         (MI_SESSION_VIEW_SIZE + \
  26                                                  MI_SESSION_POOL_SIZE + \
  27                                                  MI_SESSION_IMAGE_SIZE + \
  28                                                  MI_SESSION_WORKING_SET_SIZE)
  29
  30 #define MI_SYSTEM_VIEW_SIZE                     (32 * _1MB)
  31
  32 #define MI_HIGHEST_USER_ADDRESS                 (PVOID)0x7FFEFFFF
  33 #define MI_USER_PROBE_ADDRESS                   (PVOID)0x7FFF0000
  34 #define MI_DEFAULT_SYSTEM_RANGE_START           (PVOID)0x80000000
  35 #define MI_SYSTEM_CACHE_WS_START                (PVOID)0xC0C00000
  36 #define MI_PAGED_POOL_START                     (PVOID)0xE1000000
  37 #define MI_NONPAGED_POOL_END                    (PVOID)0xFFBE0000
  38 #define MI_DEBUG_MAPPING                        (PVOID)0xFFBFF000
  39
  40 #define MI_SYSTEM_PTE_BASE                      (PVOID)MiAddressToPte(NULL)
  41
  42 #define MI_MIN_SECONDARY_COLORS                 8
  43 #define MI_SECONDARY_COLORS                     64
  44 #define MI_MAX_SECONDARY_COLORS                 1024
  45
  46 #define MI_MIN_ALLOCATION_FRAGMENT              (4 * _1KB)
  47 #define MI_ALLOCATION_FRAGMENT                  (64 * _1KB)
  48 #define MI_MAX_ALLOCATION_FRAGMENT              (2  * _1MB)
  49
  50 #define MM_HIGHEST_VAD_ADDRESS \
  51     (PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (16 * PAGE_SIZE))
  52 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
  53
  54 #define MI_DEFAULT_SYSTEM_PTE_COUNT             50000
  55
  56 #endif /* !_M_AMD64 */
  57
  58 /* Make the code cleaner with some definitions for size multiples */
  59 #define _1KB (1024u)
  60 #define _1MB (1024 * _1KB)
  61 #define _1GB (1024 * _1MB)
  62
  63 /* Everyone loves 64K */
  64 #define _64K (64 * _1KB)
  65
  66 /* Area mapped by a PDE */
  67 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
  68
  69 /* Size of a page table */
  70 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
  71
  72 /* Size of a page directory */
  73 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
  74
  75 /* Size of all page directories for a process */
  76 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
  77
  78 /* Architecture specific count of PDEs in a directory, and count of PTEs in a PT */
  79 #ifdef _M_IX86
  80 #define PD_COUNT  1
  81 #define PDE_COUNT 1024
  82 #define PTE_COUNT 1024
  83 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
  84 #define MiIsPteOnPdeBoundary(PointerPte) \
  85     ((((ULONG_PTR)PointerPte) & (PAGE_SIZE - 1)) == 0)
  86 #elif _M_ARM
  87 #define PD_COUNT  1
  88 #define PDE_COUNT 4096
  89 #define PTE_COUNT 256
  90 #else
  91 #define PD_COUNT  PPE_PER_PAGE
  92 #define PDE_COUNT PDE_PER_PAGE
  93 #define PTE_COUNT PTE_PER_PAGE
  94 #endif
  95
  96 //
  97 // Protection Bits part of the internal memory manager Protection Mask, from:
  98 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
  99 // and public assertions.
 100 //
 101 #define MM_ZERO_ACCESS         0
 102 #define MM_READONLY            1
 103 #define MM_EXECUTE             2
 104 #define MM_EXECUTE_READ        3
 105 #define MM_READWRITE           4
 106 #define MM_WRITECOPY           5
 107 #define MM_EXECUTE_READWRITE   6
 108 #define MM_EXECUTE_WRITECOPY   7
 109 #define MM_NOCACHE             8
 110 #define MM_DECOMMIT            0x10
 111 #define MM_NOACCESS            (MM_DECOMMIT | MM_NOCACHE)
 112 #define MM_INVALID_PROTECTION  0xFFFFFFFF
 113
 114 //
 115 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
 116 // The Memory Manager's definition define the attributes that must be preserved
 117 // and these PTE definitions describe the attributes in the hardware sense. This
 118 // helps deal with hardware differences between the actual boolean expression of
 119 // the argument.
 120 //
 121 // For example, in the logical attributes, we want to express read-only as a flag
 122 // but on x86, it is writability that must be set. On the other hand, on x86, just
 123 // like in the kernel, it is disabling the caches that requires a special flag,
 124 // while on certain architectures such as ARM, it is enabling the cache which
 125 // requires a flag.
 126 //
 127 #if defined(_M_IX86) || defined(_M_AMD64)
 128 //
 129 // Access Flags
 130 //
 131 #define PTE_READONLY            0 // Doesn't exist on x86
 132 #define PTE_EXECUTE             0 // Not worrying about NX yet
 133 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 134 #define PTE_READWRITE           0x2
 135 #define PTE_WRITECOPY           0x200
 136 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
 137 #define PTE_EXECUTE_WRITECOPY   0x200
 138 #define PTE_PROTOTYPE           0x400
 139
 140 //
 141 // State Flags
 142 //
 143 #define PTE_VALID               0x1
 144 #define PTE_ACCESSED            0x20
 145 #define PTE_DIRTY               0x40
 146
 147 //
 148 // Cache flags
 149 //
 150 #define PTE_ENABLE_CACHE        0
 151 #define PTE_DISABLE_CACHE       0x10
 152 #define PTE_WRITECOMBINED_CACHE 0x10
 153 #elif defined(_M_ARM)
 154 #define PTE_READONLY            0x200
 155 #define PTE_EXECUTE             0 // Not worrying about NX yet
 156 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 157 #define PTE_READWRITE           0 // Doesn't exist on ARM
 158 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
 159 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
 160 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
 161 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
 162 //
 163 // Cache flags
 164 //
 165 #define PTE_ENABLE_CACHE        0
 166 #define PTE_DISABLE_CACHE       0x10
 167 #define PTE_WRITECOMBINED_CACHE 0x10
 168 #else
 169 #error Define these please!
 170 #endif
 171
 172 extern const ULONG_PTR MmProtectToPteMask[32];
 173 extern const ULONG MmProtectToValue[32];
 174
 175 //
 176 // Assertions for session images, addresses, and PTEs
 177 //
 178 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
 179     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
 180
 181 #define MI_IS_SESSION_ADDRESS(Address) \
 182     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
 183
 184 #define MI_IS_SESSION_PTE(Pte) \
 185     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
 186
 187 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
 188     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
 189
 190 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
 191     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
 192
 193 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
 194     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
 195
 196 //
 197 // Corresponds to MMPTE_SOFTWARE.Protection
 198 //
 199 #ifdef _M_IX86
 200 #define MM_PTE_SOFTWARE_PROTECTION_BITS   5
 201 #elif _M_ARM
 202 #define MM_PTE_SOFTWARE_PROTECTION_BITS   6
 203 #elif _M_AMD64
 204 #define MM_PTE_SOFTWARE_PROTECTION_BITS   5
 205 #else
 206 #error Define these please!
 207 #endif
 208
 209 //
 210 // Creates a software PTE with the given protection
 211 //
 212 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
 213
 214 //
 215 // Marks a PTE as deleted
 216 //
 217 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
 218 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
 219
 220 //
 221 // Special values for LoadedImports
 222 //
 223 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
 224 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
 225 #define MM_SYSLDR_SINGLE_ENTRY 0x1
 226
 227 //
 228 // Number of initial session IDs
 229 //
 230 #define MI_INITIAL_SESSION_IDS  64
 231
 232 #if defined(_M_IX86) || defined(_M_ARM)
 233 //
 234 // PFN List Sentinel
 235 //
 236 #define LIST_HEAD 0xFFFFFFFF
 237
 238 //
 239 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
 240 // we need a manual definition suited to the number of bits in the PteFrame.
 241 // This is used as a LIST_HEAD for the colored list
 242 //
 243 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
 244 #elif defined(_M_AMD64)
 245 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
 246 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
 247 #else
 248 #error Define these please!
 249 #endif
 250
 251 //
 252 // Special IRQL value (found in assertions)
 253 //
 254 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 255
 256 //
 257 // Returns the color of a page
 258 //
 259 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
 260 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
 261 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
 262
 263 #ifndef _M_AMD64
 264 //
 265 // Decodes a Prototype PTE into the underlying PTE
 266 //
 267 #define MiProtoPteToPte(x)                  \
 268     (PMMPTE)((ULONG_PTR)MmPagedPoolStart +  \
 269              (((x)->u.Proto.ProtoAddressHigh << 9) | (x)->u.Proto.ProtoAddressLow << 2))
 270
 271 //
 272 // Decodes a Prototype PTE into the underlying PTE
 273 //
 274 #define MiSubsectionPteToSubsection(x)                              \
 275     ((x)->u.Subsect.WhichPool == PagedPool) ?                       \
 276         (PMMPTE)((ULONG_PTR)MmSubsectionBase +                      \
 277                  (((x)->u.Subsect.SubsectionAddressHigh << 7) |     \
 278                    (x)->u.Subsect.SubsectionAddressLow << 3)) :     \
 279         (PMMPTE)((ULONG_PTR)MmNonPagedPoolEnd -                     \
 280                 (((x)->u.Subsect.SubsectionAddressHigh << 7) |      \
 281                   (x)->u.Subsect.SubsectionAddressLow << 3))
 282 #endif
 283
 284 //
 285 // Prototype PTEs that don't yet have a pagefile association
 286 //
 287 #ifdef _M_AMD64
 288 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
 289 #else
 290 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
 291 #endif
 292
 293 //
 294 // Number of session lists in the MM_SESSIONS_SPACE structure
 295 //
 296 #if defined(_M_AMD64)
 297 #define SESSION_POOL_LOOKASIDES 21
 298 #elif defined(_M_IX86)
 299 #define SESSION_POOL_LOOKASIDES 26
 300 #else
 301 #error Not Defined!
 302 #endif
 303
 304 //
 305 // Number of session data and tag pages
 306 //
 307 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 308 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 309
 310 //
 311 // Used by MiCheckSecuredVad
 312 //
 313 #define MM_READ_WRITE_ALLOWED   11
 314 #define MM_READ_ONLY_ALLOWED    10
 315 #define MM_NO_ACCESS_ALLOWED    01
 316 #define MM_DELETE_CHECK         85
 317
 318 //
 319 // System views are binned into 64K chunks
 320 //
 321 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
 322
 323 //
 324 // FIXFIX: These should go in ex.h after the pool merge
 325 //
 326 #ifdef _M_AMD64
 327 #define POOL_BLOCK_SIZE 16
 328 #else
 329 #define POOL_BLOCK_SIZE  8
 330 #endif
 331 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
 332 #define BASE_POOL_TYPE_MASK 1
 333 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
 334
 335 //
 336 // Pool debugging/analysis/tracing flags
 337 //
 338 #define POOL_FLAG_CHECK_TIMERS 0x1
 339 #define POOL_FLAG_CHECK_WORKERS 0x2
 340 #define POOL_FLAG_CHECK_RESOURCES 0x4
 341 #define POOL_FLAG_VERIFIER 0x8
 342 #define POOL_FLAG_CHECK_DEADLOCK 0x10
 343 #define POOL_FLAG_SPECIAL_POOL 0x20
 344 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
 345 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
 346
 347 //
 348 // BAD_POOL_HEADER codes during pool bugcheck
 349 //
 350 #define POOL_CORRUPTED_LIST 3
 351 #define POOL_SIZE_OR_INDEX_MISMATCH 5
 352 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
 353 #define POOL_HEADER_NOT_ALIGNED 7
 354 #define POOL_HEADER_IS_ZERO 8
 355 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
 356 #define POOL_ENTRY_NOT_FOUND 10
 357
 358 //
 359 // BAD_POOL_CALLER codes during pool bugcheck
 360 //
 361 #define POOL_ENTRY_CORRUPTED 1
 362 #define POOL_ENTRY_ALREADY_FREE 6
 363 #define POOL_ENTRY_NOT_ALLOCATED 7
 364 #define POOL_ALLOC_IRQL_INVALID 8
 365 #define POOL_FREE_IRQL_INVALID 9
 366 #define POOL_BILLED_PROCESS_INVALID 13
 367 #define POOL_HEADER_SIZE_INVALID 32
 368
 369 typedef struct _POOL_DESCRIPTOR
 370 {
 371     POOL_TYPE PoolType;
 372     ULONG PoolIndex;
 373     ULONG RunningAllocs;
 374     ULONG RunningDeAllocs;
 375     ULONG TotalPages;
 376     ULONG TotalBigPages;
 377     ULONG Threshold;
 378     PVOID LockAddress;
 379     PVOID PendingFrees;
 380     LONG PendingFreeDepth;
 381     SIZE_T TotalBytes;
 382     SIZE_T Spare0;
 383     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
 384 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 385
 386 typedef struct _POOL_HEADER
 387 {
 388     union
 389     {
 390         struct
 391         {
 392 #ifdef _M_AMD64
 393             USHORT PreviousSize:8;
 394             USHORT PoolIndex:8;
 395             USHORT BlockSize:8;
 396             USHORT PoolType:8;
 397 #else
 398             USHORT PreviousSize:9;
 399             USHORT PoolIndex:7;
 400             USHORT BlockSize:9;
 401             USHORT PoolType:7;
 402 #endif
 403         };
 404         ULONG Ulong1;
 405     };
 406 #ifdef _M_AMD64
 407     ULONG PoolTag;
 408 #endif
 409     union
 410     {
 411 #ifdef _M_AMD64
 412         PEPROCESS ProcessBilled;
 413 #else
 414         ULONG PoolTag;
 415 #endif
 416         struct
 417         {
 418             USHORT AllocatorBackTraceIndex;
 419             USHORT PoolTagHash;
 420         };
 421     };
 422 } POOL_HEADER, *PPOOL_HEADER;
 423
 424 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
 425 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
 426
 427 typedef struct _POOL_TRACKER_TABLE
 428 {
 429     ULONG Key;
 430     LONG NonPagedAllocs;
 431     LONG NonPagedFrees;
 432     SIZE_T NonPagedBytes;
 433     LONG PagedAllocs;
 434     LONG PagedFrees;
 435     SIZE_T PagedBytes;
 436 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
 437
 438 typedef struct _POOL_TRACKER_BIG_PAGES
 439 {
 440     PVOID Va;
 441     ULONG Key;
 442     ULONG NumberOfPages;
 443     PVOID QuotaObject;
 444 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
 445
 446 extern ULONG ExpNumberOfPagedPools;
 447 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
 448 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
 449 extern PPOOL_TRACKER_TABLE PoolTrackTable;
 450
 451 //
 452 // END FIXFIX
 453 //
 454
 455 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
 456 {
 457     LIST_ENTRY Links;
 458     UNICODE_STRING BaseName;
 459 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
 460
 461 typedef enum _MMSYSTEM_PTE_POOL_TYPE
 462 {
 463     SystemPteSpace,
 464     NonPagedPoolExpansion,
 465     MaximumPtePoolTypes
 466 } MMSYSTEM_PTE_POOL_TYPE;
 467
 468 typedef enum _MI_PFN_CACHE_ATTRIBUTE
 469 {
 470     MiNonCached,
 471     MiCached,
 472     MiWriteCombined,
 473     MiNotMapped
 474 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
 475
 476 typedef struct _PHYSICAL_MEMORY_RUN
 477 {
 478     PFN_NUMBER BasePage;
 479     PFN_NUMBER PageCount;
 480 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
 481
 482 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
 483 {
 484     ULONG NumberOfRuns;
 485     PFN_NUMBER NumberOfPages;
 486     PHYSICAL_MEMORY_RUN Run[1];
 487 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
 488
 489 typedef struct _MMCOLOR_TABLES
 490 {
 491     PFN_NUMBER Flink;
 492     PVOID Blink;
 493     PFN_NUMBER Count;
 494 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 495
 496 typedef struct _MI_LARGE_PAGE_RANGES
 497 {
 498     PFN_NUMBER StartFrame;
 499     PFN_NUMBER LastFrame;
 500 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
 501
 502 typedef struct _MMVIEW
 503 {
 504     ULONG_PTR Entry;
 505     PCONTROL_AREA ControlArea;
 506 } MMVIEW, *PMMVIEW;
 507
 508 typedef struct _MMSESSION
 509 {
 510     KGUARDED_MUTEX SystemSpaceViewLock;
 511     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
 512     PCHAR SystemSpaceViewStart;
 513     PMMVIEW SystemSpaceViewTable;
 514     ULONG SystemSpaceHashSize;
 515     ULONG SystemSpaceHashEntries;
 516     ULONG SystemSpaceHashKey;
 517     ULONG BitmapFailures;
 518     PRTL_BITMAP SystemSpaceBitMap;
 519 } MMSESSION, *PMMSESSION;
 520
 521 typedef struct _MM_SESSION_SPACE_FLAGS
 522 {
 523     ULONG Initialized:1;
 524     ULONG DeletePending:1;
 525     ULONG Filler:30;
 526 } MM_SESSION_SPACE_FLAGS;
 527
 528 typedef struct _MM_SESSION_SPACE
 529 {
 530     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
 531     LONG ReferenceCount;
 532     union
 533     {
 534         ULONG LongFlags;
 535         MM_SESSION_SPACE_FLAGS Flags;
 536     } u;
 537     ULONG SessionId;
 538     LIST_ENTRY ProcessList;
 539     LARGE_INTEGER LastProcessSwappedOutTime;
 540     PFN_NUMBER SessionPageDirectoryIndex;
 541     SIZE_T NonPageablePages;
 542     SIZE_T CommittedPages;
 543     PVOID PagedPoolStart;
 544     PVOID PagedPoolEnd;
 545     PMMPTE PagedPoolBasePde;
 546     ULONG Color;
 547     LONG ResidentProcessCount;
 548     ULONG SessionPoolAllocationFailures[4];
 549     LIST_ENTRY ImageList;
 550     LCID LocaleId;
 551     ULONG AttachCount;
 552     KEVENT AttachEvent;
 553     PEPROCESS LastProcess;
 554     LONG ProcessReferenceToSession;
 555     LIST_ENTRY WsListEntry;
 556     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
 557     MMSESSION Session;
 558     KGUARDED_MUTEX PagedPoolMutex;
 559     MM_PAGED_POOL_INFO PagedPoolInfo;
 560     MMSUPPORT Vm;
 561     PMMWSLE Wsle;
 562     PDRIVER_UNLOAD Win32KDriverUnload;
 563     POOL_DESCRIPTOR PagedPool;
 564 #if defined (_M_AMD64)
 565     MMPTE PageDirectory;
 566 #else
 567     PMMPTE PageTables;
 568 #endif
 569 #if defined (_M_AMD64)
 570     PMMPTE SpecialPoolFirstPte;
 571     PMMPTE SpecialPoolLastPte;
 572     PMMPTE NextPdeForSpecialPoolExpansion;
 573     PMMPTE LastPdeForSpecialPoolExpansion;
 574     PFN_NUMBER SpecialPagesInUse;
 575 #endif
 576     LONG ImageLoadingCount;
 577 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
 578
 579 extern PMM_SESSION_SPACE MmSessionSpace;
 580 extern MMPTE HyperTemplatePte;
 581 extern MMPDE ValidKernelPde;
 582 extern MMPTE ValidKernelPte;
 583 extern MMPDE ValidKernelPdeLocal;
 584 extern MMPTE ValidKernelPteLocal;
 585 extern MMPDE DemandZeroPde;
 586 extern MMPTE DemandZeroPte;
 587 extern MMPTE PrototypePte;
 588 extern MMPTE MmDecommittedPte;
 589 extern BOOLEAN MmLargeSystemCache;
 590 extern BOOLEAN MmZeroPageFile;
 591 extern BOOLEAN MmProtectFreedNonPagedPool;
 592 extern BOOLEAN MmTrackLockedPages;
 593 extern BOOLEAN MmTrackPtes;
 594 extern BOOLEAN MmDynamicPfn;
 595 extern BOOLEAN MmMirroring;
 596 extern BOOLEAN MmMakeLowMemory;
 597 extern BOOLEAN MmEnforceWriteProtection;
 598 extern SIZE_T MmAllocationFragment;
 599 extern ULONG MmConsumedPoolPercentage;
 600 extern ULONG MmVerifyDriverBufferType;
 601 extern ULONG MmVerifyDriverLevel;
 602 extern WCHAR MmVerifyDriverBuffer[512];
 603 extern WCHAR MmLargePageDriverBuffer[512];
 604 extern LIST_ENTRY MiLargePageDriverList;
 605 extern BOOLEAN MiLargePageAllDrivers;
 606 extern ULONG MmVerifyDriverBufferLength;
 607 extern ULONG MmLargePageDriverBufferLength;
 608 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
 609 extern SIZE_T MmMaximumNonPagedPoolInBytes;
 610 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
 611 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
 612 extern PVOID MmNonPagedSystemStart;
 613 extern SIZE_T MiNonPagedSystemSize;
 614 extern PVOID MmNonPagedPoolStart;
 615 extern PVOID MmNonPagedPoolExpansionStart;
 616 extern PVOID MmNonPagedPoolEnd;
 617 extern SIZE_T MmSizeOfPagedPoolInBytes;
 618 extern PVOID MmPagedPoolStart;
 619 extern PVOID MmPagedPoolEnd;
 620 extern PVOID MmSessionBase;
 621 extern SIZE_T MmSessionSize;
 622 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
 623 extern PMMPTE MiFirstReservedZeroingPte;
 624 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
 625 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
 626 extern SIZE_T MmBootImageSize;
 627 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
 628 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
 629 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
 630 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
 631 extern ULONG_PTR MxPfnAllocation;
 632 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
 633 extern RTL_BITMAP MiPfnBitMap;
 634 extern KGUARDED_MUTEX MmPagedPoolMutex;
 635 extern KGUARDED_MUTEX MmSectionCommitMutex;
 636 extern PVOID MmPagedPoolStart;
 637 extern PVOID MmPagedPoolEnd;
 638 extern PVOID MmNonPagedSystemStart;
 639 extern PVOID MiSystemViewStart;
 640 extern SIZE_T MmSystemViewSize;
 641 extern PVOID MmSessionBase;
 642 extern PVOID MiSessionSpaceEnd;
 643 extern PMMPTE MiSessionImagePteStart;
 644 extern PMMPTE MiSessionImagePteEnd;
 645 extern PMMPTE MiSessionBasePte;
 646 extern PMMPTE MiSessionLastPte;
 647 extern SIZE_T MmSizeOfPagedPoolInBytes;
 648 extern PMMPDE MmSystemPagePtes;
 649 extern PVOID MmSystemCacheStart;
 650 extern PVOID MmSystemCacheEnd;
 651 extern MMSUPPORT MmSystemCacheWs;
 652 extern SIZE_T MmAllocatedNonPagedPool;
 653 extern ULONG MmSpecialPoolTag;
 654 extern PVOID MmHyperSpaceEnd;
 655 extern PMMWSL MmSystemCacheWorkingSetList;
 656 extern SIZE_T MmMinimumNonPagedPoolSize;
 657 extern ULONG MmMinAdditionNonPagedPoolPerMb;
 658 extern SIZE_T MmDefaultMaximumNonPagedPool;
 659 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
 660 extern ULONG MmSecondaryColors;
 661 extern ULONG MmSecondaryColorMask;
 662 extern ULONG MmNumberOfSystemPtes;
 663 extern ULONG MmMaximumNonPagedPoolPercent;
 664 extern ULONG MmLargeStackSize;
 665 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
 666 extern MMPFNLIST MmStandbyPageListByPriority[8];
 667 extern ULONG MmProductType;
 668 extern MM_SYSTEMSIZE MmSystemSize;
 669 extern PKEVENT MiLowMemoryEvent;
 670 extern PKEVENT MiHighMemoryEvent;
 671 extern PKEVENT MiLowPagedPoolEvent;
 672 extern PKEVENT MiHighPagedPoolEvent;
 673 extern PKEVENT MiLowNonPagedPoolEvent;
 674 extern PKEVENT MiHighNonPagedPoolEvent;
 675 extern PFN_NUMBER MmLowMemoryThreshold;
 676 extern PFN_NUMBER MmHighMemoryThreshold;
 677 extern PFN_NUMBER MiLowPagedPoolThreshold;
 678 extern PFN_NUMBER MiHighPagedPoolThreshold;
 679 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
 680 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 681 extern PFN_NUMBER MmMinimumFreePages;
 682 extern PFN_NUMBER MmPlentyFreePages;
 683 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 684 extern PFN_NUMBER MmResidentAvailablePages;
 685 extern PFN_NUMBER MmResidentAvailableAtInit;
 686 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
 687 extern PFN_NUMBER MmTotalSystemDriverPages;
 688 extern PVOID MiSessionImageStart;
 689 extern PVOID MiSessionImageEnd;
 690 extern PMMPTE MiHighestUserPte;
 691 extern PMMPDE MiHighestUserPde;
 692 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
 693 extern PMMPTE MmSharedUserDataPte;
 694 extern LIST_ENTRY MmProcessList;
 695 extern BOOLEAN MmZeroingPageThreadActive;
 696 extern KEVENT MmZeroingPageEvent;
 697 extern ULONG MmSystemPageColor;
 698 extern ULONG MmProcessColorSeed;
 699 extern PMMWSL MmWorkingSetList;
 700 extern PFN_NUMBER MiNumberOfFreePages;
 701 extern SIZE_T MmSessionViewSize;
 702 extern SIZE_T MmSessionPoolSize;
 703 extern SIZE_T MmSessionImageSize;
 704 extern PVOID MiSystemViewStart;
 705 extern PVOID MiSessionPoolEnd;     // 0xBE000000
 706 extern PVOID MiSessionPoolStart;   // 0xBD000000
 707 extern PVOID MiSessionViewStart;   // 0xBE000000
 708 extern PVOID MiSessionSpaceWs;
 709 extern ULONG MmMaximumDeadKernelStacks;
 710 extern SLIST_HEADER MmDeadStackSListHead;
 711 extern MM_AVL_TABLE MmSectionBasedRoot;
 712 extern KGUARDED_MUTEX MmSectionBasedMutex;
 713 extern PVOID MmHighSectionBase;
 714 extern SIZE_T MmSystemLockPagesCount;
 715 extern ULONG_PTR MmSubsectionBase;
 716 extern LARGE_INTEGER MmCriticalSectionTimeout;
 717
 718 BOOLEAN
 719 FORCEINLINE
 720 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
 721 {
 722     return ((MemoryType == LoaderFree) ||
 723             (MemoryType == LoaderLoadedProgram) ||
 724             (MemoryType == LoaderFirmwareTemporary) ||
 725             (MemoryType == LoaderOsloaderStack));
 726 }
 727
 728 BOOLEAN
 729 FORCEINLINE
 730 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
 731 {
 732     return ((MemoryType == LoaderFirmwarePermanent) ||
 733             (MemoryType == LoaderSpecialMemory) ||
 734             (MemoryType == LoaderHALCachedMemory) ||
 735             (MemoryType == LoaderBBTMemory));
 736 }
 737
 738 #ifdef _M_AMD64
 739 BOOLEAN
 740 FORCEINLINE
 741 MiIsUserPxe(PVOID Address)
 742 {
 743     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
 744 }
 745
 746 BOOLEAN
 747 FORCEINLINE
 748 MiIsUserPpe(PVOID Address)
 749 {
 750     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
 751 }
 752
 753 BOOLEAN
 754 FORCEINLINE
 755 MiIsUserPde(PVOID Address)
 756 {
 757     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
 758 }
 759
 760 BOOLEAN
 761 FORCEINLINE
 762 MiIsUserPte(PVOID Address)
 763 {
 764     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
 765 }
 766 #else
 767 BOOLEAN
 768 FORCEINLINE
 769 MiIsUserPde(PVOID Address)
 770 {
 771     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
 772             (Address <= (PVOID)MiHighestUserPde));
 773 }
 774
 775 BOOLEAN
 776 FORCEINLINE
 777 MiIsUserPte(PVOID Address)
 778 {
 779     return (Address <= (PVOID)MiHighestUserPte);
 780 }
 781 #endif
 782
 783 //
 784 // Figures out the hardware bits for a PTE
 785 //
 786 ULONG_PTR
 787 FORCEINLINE
 788 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
 789 {
 790     MMPTE TempPte;
 791
 792     /* Start fresh */
 793     TempPte.u.Long = 0;
 794
 795     /* Make it valid and accessed */
 796     TempPte.u.Hard.Valid = TRUE;
 797     MI_MAKE_ACCESSED_PAGE(&TempPte);
 798
 799     /* Is this for user-mode? */
 800     if (
 801 #if (_MI_PAGING_LEVELS == 4)
 802         MiIsUserPxe(PointerPte) ||
 803 #endif
 804 #if (_MI_PAGING_LEVELS >= 3)
 805         MiIsUserPpe(PointerPte) ||
 806 #endif
 807         MiIsUserPde(PointerPte) ||
 808         MiIsUserPte(PointerPte))
 809     {
 810         /* Set the owner bit */
 811         MI_MAKE_OWNER_PAGE(&TempPte);
 812     }
 813
 814     /* FIXME: We should also set the global bit */
 815
 816     /* Return the protection */
 817     return TempPte.u.Long;
 818 }
 819
 820 //
 821 // Creates a valid kernel PTE with the given protection
 822 //
 823 FORCEINLINE
 824 VOID
 825 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
 826                             IN PMMPTE MappingPte,
 827                             IN ULONG_PTR ProtectionMask,
 828                             IN PFN_NUMBER PageFrameNumber)
 829 {
 830     /* Only valid for kernel, non-session PTEs */
 831     ASSERT(MappingPte > MiHighestUserPte);
 832     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
 833     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 834
 835     /* Start fresh */
 836     *NewPte = ValidKernelPte;
 837
 838     /* Set the protection and page */
 839     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 840     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 841 }
 842
 843 //
 844 // Creates a valid PTE with the given protection
 845 //
 846 FORCEINLINE
 847 VOID
 848 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
 849                      IN PMMPTE MappingPte,
 850                      IN ULONG_PTR ProtectionMask,
 851                      IN PFN_NUMBER PageFrameNumber)
 852 {
 853     /* Set the protection and page */
 854     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
 855     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 856     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 857 }
 858
 859 //
 860 // Creates a valid user PTE with the given protection
 861 //
 862 FORCEINLINE
 863 VOID
 864 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
 865                           IN PMMPTE MappingPte,
 866                           IN ULONG_PTR ProtectionMask,
 867                           IN PFN_NUMBER PageFrameNumber)
 868 {
 869     /* Only valid for kernel, non-session PTEs */
 870     ASSERT(MappingPte <= MiHighestUserPte);
 871
 872     /* Start fresh */
 873     *NewPte = ValidKernelPte;
 874
 875     /* Set the protection and page */
 876     NewPte->u.Hard.Owner = TRUE;
 877     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 878     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 879 }
 880
 881 #ifndef _M_AMD64
 882 //
 883 // Builds a Prototype PTE for the address of the PTE
 884 //
 885 FORCEINLINE
 886 VOID
 887 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
 888                       IN PMMPTE PointerPte)
 889 {
 890     ULONG_PTR Offset;
 891
 892     /* Mark this as a prototype */
 893     NewPte->u.Long = 0;
 894     NewPte->u.Proto.Prototype = 1;
 895
 896     /*
 897      * Prototype PTEs are only valid in paged pool by design, this little trick
 898      * lets us only use 30 bits for the adress of the PTE, as long as the area
 899      * stays 1024MB At most.
 900      */
 901     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 902
 903     /*
 904      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
 905      * and the other 21 bits go in the "high"
 906      */
 907     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
 908     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
 909 }
 910
 911 //
 912 // Builds a Subsection PTE for the address of the Segment
 913 //
 914 FORCEINLINE
 915 VOID
 916 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
 917                        IN PVOID Segment)
 918 {
 919     ULONG_PTR Offset;
 920
 921     /* Mark this as a prototype */
 922     NewPte->u.Long = 0;
 923     NewPte->u.Subsect.Prototype = 1;
 924
 925     /*
 926      * Segments are only valid either in nonpaged pool. We store the 20 bit
 927      * difference either from the top or bottom of nonpaged pool, giving a
 928      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
 929      * 256MB.
 930      */
 931     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
 932     {
 933         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
 934         NewPte->u.Subsect.WhichPool = PagedPool;
 935     }
 936     else
 937     {
 938         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
 939         NewPte->u.Subsect.WhichPool = NonPagedPool;
 940     }
 941
 942     /*
 943      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
 944      * and the other 20 bits go in the "high"
 945      */
 946     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
 947     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
 948 }
 949
 950 #endif
 951
 952 //
 953 // Returns if the page is physically resident (ie: a large page)
 954 // FIXFIX: CISC/x86 only?
 955 //
 956 FORCEINLINE
 957 BOOLEAN
 958 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
 959 {
 960     PMMPDE PointerPde;
 961
 962     /* Large pages are never paged out, always physically resident */
 963     PointerPde = MiAddressToPde(Address);
 964     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
 965 }
 966
 967 //
 968 // Writes a valid PTE
 969 //
 970 VOID
 971 FORCEINLINE
 972 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
 973                    IN MMPTE TempPte)
 974 {
 975     /* Write the valid PTE */
 976     ASSERT(PointerPte->u.Hard.Valid == 0);
 977     ASSERT(TempPte.u.Hard.Valid == 1);
 978     *PointerPte = TempPte;
 979 }
 980
 981 //
 982 // Writes an invalid PTE
 983 //
 984 VOID
 985 FORCEINLINE
 986 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
 987                      IN MMPTE InvalidPte)
 988 {
 989     /* Write the invalid PTE */
 990     ASSERT(InvalidPte.u.Hard.Valid == 0);
 991     *PointerPte = InvalidPte;
 992 }
 993
 994 //
 995 // Writes a valid PDE
 996 //
 997 VOID
 998 FORCEINLINE
 999 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1000                    IN MMPDE TempPde)
1001 {
1002     /* Write the valid PDE */
1003     ASSERT(PointerPde->u.Hard.Valid == 0);
1004     ASSERT(TempPde.u.Hard.Valid == 1);
1005     *PointerPde = TempPde;
1006 }
1007
1008 //
1009 // Writes an invalid PDE
1010 //
1011 VOID
1012 FORCEINLINE
1013 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1014                      IN MMPDE InvalidPde)
1015 {
1016     /* Write the invalid PDE */
1017     ASSERT(InvalidPde.u.Hard.Valid == 0);
1018     *PointerPde = InvalidPde;
1019 }
1020
1021 //
1022 // Checks if the thread already owns a working set
1023 //
1024 FORCEINLINE
1025 BOOLEAN
1026 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1027 {
1028     /* If any of these are held, return TRUE */
1029     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1030             (Thread->OwnsProcessWorkingSetShared) ||
1031             (Thread->OwnsSystemWorkingSetExclusive) ||
1032             (Thread->OwnsSystemWorkingSetShared) ||
1033             (Thread->OwnsSessionWorkingSetExclusive) ||
1034             (Thread->OwnsSessionWorkingSetShared));
1035 }
1036
1037 //
1038 // Checks if the process owns the working set lock
1039 //
1040 FORCEINLINE
1041 BOOLEAN
1042 MI_WS_OWNER(IN PEPROCESS Process)
1043 {
1044     /* Check if this process is the owner, and that the thread owns the WS */
1045     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1046     {
1047         DPRINT1("Thread: %p is not an owner\n", PsGetCurrentThread());
1048     }
1049     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1050     {
1051         DPRINT1("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1052     }
1053     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1054             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1055              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1056 }
1057
1058 //
1059 // New ARM3<->RosMM PAGE Architecture
1060 //
1061 BOOLEAN
1062 FORCEINLINE
1063 MiIsRosSectionObject(IN PVOID Section)
1064 {
1065     PROS_SECTION_OBJECT RosSection = Section;
1066     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1067     return FALSE;
1068 }
1069
1070 #ifdef _WIN64
1071 // HACK ON TOP OF HACK ALERT!!!
1072 #define MI_GET_ROS_DATA(x) \
1073     (((x)->RosMmData == 0) ? NULL : ((PMMROSPFN)((ULONG64)(ULONG)((x)->RosMmData) | \
1074                                     ((ULONG64)MmNonPagedPoolStart & 0xffffffff00000000ULL))))
1075 #else
1076 #define MI_GET_ROS_DATA(x)   ((PMMROSPFN)(x->RosMmData))
1077 #endif
1078 #define MI_IS_ROS_PFN(x)     (((x)->u4.AweAllocation == TRUE) && (MI_GET_ROS_DATA(x) != NULL))
1079 #define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
1080 typedef struct _MMROSPFN
1081 {
1082     PMM_RMAP_ENTRY RmapListHead;
1083     SWAPENTRY SwapEntry;
1084 } MMROSPFN, *PMMROSPFN;
1085
1086 #define RosMmData            AweReferenceCount
1087
1088 VOID
1089 NTAPI
1090 MiDecrementReferenceCount(
1091     IN PMMPFN Pfn1,
1092     IN PFN_NUMBER PageFrameIndex
1093 );
1094
1095 FORCEINLINE
1096 BOOLEAN
1097 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1098 {
1099     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1100 }
1101
1102 //
1103 // Locks the working set for the given process
1104 //
1105 FORCEINLINE
1106 VOID
1107 MiLockProcessWorkingSet(IN PEPROCESS Process,
1108                         IN PETHREAD Thread)
1109 {
1110     /* Shouldn't already be owning the process working set */
1111     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1112     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1113
1114     /* Block APCs, make sure that still nothing is already held */
1115     KeEnterGuardedRegion();
1116     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1117
1118     /* Lock the working set */
1119     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1120
1121     /* Now claim that we own the lock */
1122     ASSERT(!MI_IS_WS_UNSAFE(Process));
1123     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1124     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1125 }
1126
1127 FORCEINLINE
1128 VOID
1129 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1130                               IN PETHREAD Thread)
1131 {
1132     /* Shouldn't already be owning the process working set */
1133     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1134     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1135
1136     /* Block APCs, make sure that still nothing is already held */
1137     KeEnterGuardedRegion();
1138     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1139
1140     /* Lock the working set */
1141     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1142
1143     /* Now claim that we own the lock */
1144     ASSERT(!MI_IS_WS_UNSAFE(Process));
1145     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1146     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1147     Thread->OwnsProcessWorkingSetShared = TRUE;
1148 }
1149
1150 FORCEINLINE
1151 VOID
1152 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1153                               IN PETHREAD Thread)
1154 {
1155     /* Shouldn't already be owning the process working set */
1156     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1157
1158     /* APCs must be blocked, make sure that still nothing is already held */
1159     ASSERT(KeAreAllApcsDisabled() == TRUE);
1160     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1161
1162     /* Lock the working set */
1163     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1164
1165     /* Now claim that we own the lock */
1166     ASSERT(!MI_IS_WS_UNSAFE(Process));
1167     Process->Vm.Flags.AcquiredUnsafe = 1;
1168     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1169     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1170 }
1171
1172 //
1173 // Unlocks the working set for the given process
1174 //
1175 FORCEINLINE
1176 VOID
1177 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1178                           IN PETHREAD Thread)
1179 {
1180     /* Make sure we are the owner of a safe acquisition */
1181     ASSERT(MI_WS_OWNER(Process));
1182     ASSERT(!MI_IS_WS_UNSAFE(Process));
1183
1184     /* The thread doesn't own it anymore */
1185     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1186     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1187
1188     /* Release the lock and re-enable APCs */
1189     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1190     KeLeaveGuardedRegion();
1191 }
1192
1193 //
1194 // Unlocks the working set for the given process
1195 //
1196 FORCEINLINE
1197 VOID
1198 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1199                                 IN PETHREAD Thread)
1200 {
1201     /* Make sure we are the owner of an unsafe acquisition */
1202     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1203     ASSERT(KeAreAllApcsDisabled() == TRUE);
1204     ASSERT(MI_WS_OWNER(Process));
1205     ASSERT(MI_IS_WS_UNSAFE(Process));
1206
1207     /* No longer unsafe */
1208     Process->Vm.Flags.AcquiredUnsafe = 0;
1209
1210     /* The thread doesn't own it anymore */
1211     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1212     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1213
1214     /* Release the lock but don't touch APC state */
1215     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1216     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1217 }
1218
1219 //
1220 // Locks the working set
1221 //
1222 FORCEINLINE
1223 VOID
1224 MiLockWorkingSet(IN PETHREAD Thread,
1225                  IN PMMSUPPORT WorkingSet)
1226 {
1227     /* Block APCs */
1228     KeEnterGuardedRegion();
1229
1230     /* Working set should be in global memory */
1231     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1232
1233     /* Thread shouldn't already be owning something */
1234     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1235
1236     /* Lock this working set */
1237     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1238
1239     /* Which working set is this? */
1240     if (WorkingSet == &MmSystemCacheWs)
1241     {
1242         /* Own the system working set */
1243         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1244                (Thread->OwnsSystemWorkingSetShared == FALSE));
1245         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1246     }
1247     else if (WorkingSet->Flags.SessionSpace)
1248     {
1249         /* Own the session working set */
1250         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1251                (Thread->OwnsSessionWorkingSetShared == FALSE));
1252         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1253     }
1254     else
1255     {
1256         /* Own the process working set */
1257         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1258                (Thread->OwnsProcessWorkingSetShared == FALSE));
1259         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1260     }
1261 }
1262
1263 //
1264 // Unlocks the working set
1265 //
1266 FORCEINLINE
1267 VOID
1268 MiUnlockWorkingSet(IN PETHREAD Thread,
1269                    IN PMMSUPPORT WorkingSet)
1270 {
1271     /* Working set should be in global memory */
1272     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1273
1274     /* Which working set is this? */
1275     if (WorkingSet == &MmSystemCacheWs)
1276     {
1277         /* Release the system working set */
1278         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1279                (Thread->OwnsSystemWorkingSetShared == TRUE));
1280         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1281     }
1282     else if (WorkingSet->Flags.SessionSpace)
1283     {
1284         /* Release the session working set */
1285         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1286                (Thread->OwnsSessionWorkingSetShared == TRUE));
1287         Thread->OwnsSessionWorkingSetExclusive = 0;
1288     }
1289     else
1290     {
1291         /* Release the process working set */
1292         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1293                (Thread->OwnsProcessWorkingSetShared));
1294         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1295     }
1296
1297     /* Release the working set lock */
1298     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1299
1300     /* Unblock APCs */
1301     KeLeaveGuardedRegion();
1302 }
1303
1304 FORCEINLINE
1305 VOID
1306 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1307                                   IN PETHREAD Thread,
1308                                   IN BOOLEAN Safe,
1309                                   IN BOOLEAN Shared)
1310 {
1311     ASSERT(MI_WS_OWNER(Process));
1312
1313     /* Check if the current owner is unsafe */
1314     if (MI_IS_WS_UNSAFE(Process))
1315     {
1316         /* Release unsafely */
1317         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1318         Safe = FALSE;
1319         Shared = FALSE;
1320     }
1321     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1322     {
1323         /* Owner is safe and exclusive, release normally */
1324         MiUnlockProcessWorkingSet(Process, Thread);
1325         Safe = TRUE;
1326         Shared = FALSE;
1327     }
1328     else
1329     {
1330         /* Owner is shared (implies safe), release normally */
1331         ASSERT(FALSE);
1332         Safe = TRUE;
1333         Shared = TRUE;
1334     }
1335 }
1336
1337 FORCEINLINE
1338 VOID
1339 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1340                                 IN PETHREAD Thread,
1341                                 IN BOOLEAN Safe,
1342                                 IN BOOLEAN Shared)
1343 {
1344     ASSERT(Shared == FALSE);
1345
1346     /* Check if this was a safe lock or not */
1347     if (Safe)
1348     {
1349         /* Reacquire safely */
1350         MiLockProcessWorkingSet(Process, Thread);
1351     }
1352     else
1353     {
1354         /* Reacquire unsafely */
1355         MiLockProcessWorkingSetUnsafe(Process, Thread);
1356     }
1357 }
1358
1359 //
1360 // Returns the ProtoPTE inside a VAD for the given VPN
1361 //
1362 FORCEINLINE
1363 PMMPTE
1364 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1365                              IN ULONG_PTR Vpn)
1366 {
1367     PMMPTE ProtoPte;
1368
1369     /* Find the offset within the VAD's prototype PTEs */
1370     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1371     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1372     return ProtoPte;
1373 }
1374
1375 //
1376 // Returns the PFN Database entry for the given page number
1377 // Warning: This is not necessarily a valid PFN database entry!
1378 //
1379 FORCEINLINE
1380 PMMPFN
1381 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1382 {
1383     /* Get the entry */
1384     return &MmPfnDatabase[Pfn];
1385 };
1386
1387 //
1388 // Drops a locked page without dereferencing it
1389 //
1390 FORCEINLINE
1391 VOID
1392 MiDropLockCount(IN PMMPFN Pfn1)
1393 {
1394     /* This page shouldn't be locked, but it should be valid */
1395     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1396     ASSERT(Pfn1->u2.ShareCount == 0);
1397
1398     /* Is this the last reference to the page */
1399     if (Pfn1->u3.e2.ReferenceCount == 1)
1400     {
1401         /* It better not be valid */
1402         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1403
1404         /* Is it a prototype PTE? */
1405         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1406             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1407         {
1408             /* FIXME: We should return commit */
1409             DPRINT1("Not returning commit for prototype PTE\n");
1410         }
1411
1412         /* Update the counter */
1413         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1414     }
1415 }
1416
1417 //
1418 // Drops a locked page and dereferences it
1419 //
1420 FORCEINLINE
1421 VOID
1422 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1423 {
1424     USHORT RefCount, OldRefCount;
1425     PFN_NUMBER PageFrameIndex;
1426
1427     /* Loop while we decrement the page successfully */
1428     do
1429     {
1430         /* There should be at least one reference */
1431         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1432         ASSERT(OldRefCount != 0);
1433
1434         /* Are we the last one */
1435         if (OldRefCount == 1)
1436         {
1437             /* The page shoudln't be shared not active at this point */
1438             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1439             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1440             ASSERT(Pfn1->u2.ShareCount == 0);
1441
1442             /* Is it a prototype PTE? */
1443             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1444                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1445             {
1446                 /* FIXME: We should return commit */
1447                 DPRINT1("Not returning commit for prototype PTE\n");
1448             }
1449
1450             /* Update the counter, and drop a reference the long way */
1451             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1452             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1453             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1454             return;
1455         }
1456
1457         /* Drop a reference the short way, and that's it */
1458         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1459                                                 OldRefCount - 1,
1460                                                 OldRefCount);
1461         ASSERT(RefCount != 0);
1462     } while (OldRefCount != RefCount);
1463
1464     /* If we got here, there should be more than one reference */
1465     ASSERT(RefCount > 1);
1466     if (RefCount == 2)
1467     {
1468         /* Is it still being shared? */
1469         if (Pfn1->u2.ShareCount >= 1)
1470         {
1471             /* Then it should be valid */
1472             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1473
1474             /* Is it a prototype PTE? */
1475             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1476                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1477             {
1478                 /* We don't handle ethis */
1479                 ASSERT(FALSE);
1480             }
1481
1482             /* Update the counter */
1483             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1484         }
1485     }
1486 }
1487
1488 //
1489 // References a locked page and updates the counter
1490 // Used in MmProbeAndLockPages to handle different edge cases
1491 //
1492 FORCEINLINE
1493 VOID
1494 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1495 {
1496     USHORT RefCount, OldRefCount;
1497
1498     /* Sanity check */
1499     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1500
1501     /* Does ARM3 own the page? */
1502     if (MI_IS_ROS_PFN(Pfn1))
1503     {
1504         /* ReactOS Mm doesn't track share count */
1505         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1506     }
1507     else
1508     {
1509         /* On ARM3 pages, we should see a valid share count */
1510         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1511
1512         /* Is it a prototype PTE? */
1513         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1514             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1515         {
1516             /* FIXME: We should charge commit */
1517             DPRINT1("Not charging commit for prototype PTE\n");
1518         }
1519     }
1520
1521     /* More locked pages! */
1522     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1523
1524     /* Loop trying to update the reference count */
1525     do
1526     {
1527         /* Get the current reference count, make sure it's valid */
1528         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1529         ASSERT(OldRefCount != 0);
1530         ASSERT(OldRefCount < 2500);
1531
1532         /* Bump it up by one */
1533         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1534                                                 OldRefCount + 1,
1535                                                 OldRefCount);
1536         ASSERT(RefCount != 0);
1537     } while (OldRefCount != RefCount);
1538
1539     /* Was this the first lock attempt? If not, undo our bump */
1540     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1541 }
1542
1543 //
1544 // References a locked page and updates the counter
1545 // Used in all other cases except MmProbeAndLockPages
1546 //
1547 FORCEINLINE
1548 VOID
1549 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1550 {
1551     USHORT NewRefCount;
1552
1553     /* Is it a prototype PTE? */
1554     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1555         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1556     {
1557         /* FIXME: We should charge commit */
1558         DPRINT1("Not charging commit for prototype PTE\n");
1559     }
1560
1561     /* More locked pages! */
1562     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1563
1564     /* Update the reference count */
1565     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1566     if (NewRefCount == 2)
1567     {
1568         /* Is it locked or shared? */
1569         if (Pfn1->u2.ShareCount)
1570         {
1571             /* It's shared, so make sure it's active */
1572             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1573         }
1574         else
1575         {
1576             /* It's locked, so we shouldn't lock again */
1577             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1578         }
1579     }
1580     else
1581     {
1582         /* Someone had already locked the page, so undo our bump */
1583         ASSERT(NewRefCount < 2500);
1584         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1585     }
1586 }
1587
1588 //
1589 // References a locked page and updates the counter
1590 // Used in all other cases except MmProbeAndLockPages
1591 //
1592 FORCEINLINE
1593 VOID
1594 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1595 {
1596     USHORT NewRefCount;
1597
1598     /* Make sure the page isn't used yet */
1599     ASSERT(Pfn1->u2.ShareCount == 0);
1600     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1601
1602     /* Is it a prototype PTE? */
1603     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1604         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1605     {
1606         /* FIXME: We should charge commit */
1607         DPRINT1("Not charging commit for prototype PTE\n");
1608     }
1609
1610     /* More locked pages! */
1611     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1612
1613     /* Update the reference count */
1614     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1615     if (NewRefCount != 1)
1616     {
1617         /* Someone had already locked the page, so undo our bump */
1618         ASSERT(NewRefCount < 2500);
1619         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1620     }
1621 }
1622
1623 FORCEINLINE
1624 VOID
1625 MiIncrementPageTableReferences(IN PVOID Address)
1626 {
1627     PUSHORT RefCount;
1628
1629     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1630
1631     *RefCount += 1;
1632     ASSERT(*RefCount <= PTE_PER_PAGE);
1633 }
1634
1635 FORCEINLINE
1636 VOID
1637 MiDecrementPageTableReferences(IN PVOID Address)
1638 {
1639     PUSHORT RefCount;
1640
1641     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1642
1643     *RefCount -= 1;
1644     ASSERT(*RefCount < PTE_PER_PAGE);
1645 }
1646
1647 FORCEINLINE
1648 USHORT
1649 MiQueryPageTableReferences(IN PVOID Address)
1650 {
1651     PUSHORT RefCount;
1652
1653     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1654
1655     return *RefCount;
1656 }
1657
1658 BOOLEAN
1659 NTAPI
1660 MmArmInitSystem(
1661     IN ULONG Phase,
1662     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1663 );
1664
1665 VOID
1666 NTAPI
1667 MiInitializeSessionSpaceLayout();
1668
1669 NTSTATUS
1670 NTAPI
1671 MiInitMachineDependent(
1672     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1673 );
1674
1675 VOID
1676 NTAPI
1677 MiComputeColorInformation(
1678     VOID
1679 );
1680
1681 VOID
1682 NTAPI
1683 MiMapPfnDatabase(
1684     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1685 );
1686
1687 VOID
1688 NTAPI
1689 MiInitializeColorTables(
1690     VOID
1691 );
1692
1693 VOID
1694 NTAPI
1695 MiInitializePfnDatabase(
1696     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1697 );
1698
1699 VOID
1700 NTAPI
1701 MiInitializeSessionIds(
1702     VOID
1703 );
1704
1705 BOOLEAN
1706 NTAPI
1707 MiInitializeMemoryEvents(
1708     VOID
1709 );
1710
1711 PFN_NUMBER
1712 NTAPI
1713 MxGetNextPage(
1714     IN PFN_NUMBER PageCount
1715 );
1716
1717 PPHYSICAL_MEMORY_DESCRIPTOR
1718 NTAPI
1719 MmInitializeMemoryLimits(
1720     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1721     IN PBOOLEAN IncludeType
1722 );
1723
1724 PFN_NUMBER
1725 NTAPI
1726 MiPagesInLoaderBlock(
1727     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1728     IN PBOOLEAN IncludeType
1729 );
1730
1731 VOID
1732 FASTCALL
1733 MiSyncARM3WithROS(
1734     IN PVOID AddressStart,
1735     IN PVOID AddressEnd
1736 );
1737
1738 NTSTATUS
1739 NTAPI
1740 MiRosProtectVirtualMemory(
1741     IN PEPROCESS Process,
1742     IN OUT PVOID *BaseAddress,
1743     IN OUT PSIZE_T NumberOfBytesToProtect,
1744     IN ULONG NewAccessProtection,
1745     OUT PULONG OldAccessProtection OPTIONAL
1746 );
1747
1748 NTSTATUS
1749 NTAPI
1750 MmArmAccessFault(
1751     IN BOOLEAN StoreInstruction,
1752     IN PVOID Address,
1753     IN KPROCESSOR_MODE Mode,
1754     IN PVOID TrapInformation
1755 );
1756
1757 NTSTATUS
1758 FASTCALL
1759 MiCheckPdeForPagedPool(
1760     IN PVOID Address
1761 );
1762
1763 VOID
1764 NTAPI
1765 MiInitializeNonPagedPool(
1766     VOID
1767 );
1768
1769 VOID
1770 NTAPI
1771 MiInitializeNonPagedPoolThresholds(
1772     VOID
1773 );
1774
1775 VOID
1776 NTAPI
1777 MiInitializePoolEvents(
1778     VOID
1779 );
1780
1781 VOID                      //
1782 NTAPI                     //
1783 InitializePool(           //
1784     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1785     IN ULONG Threshold    //
1786 );                        //
1787
1788 // FIXFIX: THIS ONE TOO
1789 VOID
1790 NTAPI
1791 INIT_FUNCTION
1792 ExInitializePoolDescriptor(
1793     IN PPOOL_DESCRIPTOR PoolDescriptor,
1794     IN POOL_TYPE PoolType,
1795     IN ULONG PoolIndex,
1796     IN ULONG Threshold,
1797     IN PVOID PoolLock
1798 );
1799
1800 NTSTATUS
1801 NTAPI
1802 MiInitializeSessionPool(
1803     VOID
1804 );
1805
1806 VOID
1807 NTAPI
1808 MiInitializeSystemPtes(
1809     IN PMMPTE StartingPte,
1810     IN ULONG NumberOfPtes,
1811     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1812 );
1813
1814 PMMPTE
1815 NTAPI
1816 MiReserveSystemPtes(
1817     IN ULONG NumberOfPtes,
1818     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1819 );
1820
1821 VOID
1822 NTAPI
1823 MiReleaseSystemPtes(
1824     IN PMMPTE StartingPte,
1825     IN ULONG NumberOfPtes,
1826     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1827 );
1828
1829
1830 PFN_NUMBER
1831 NTAPI
1832 MiFindContiguousPages(
1833     IN PFN_NUMBER LowestPfn,
1834     IN PFN_NUMBER HighestPfn,
1835     IN PFN_NUMBER BoundaryPfn,
1836     IN PFN_NUMBER SizeInPages,
1837     IN MEMORY_CACHING_TYPE CacheType
1838 );
1839
1840 PVOID
1841 NTAPI
1842 MiCheckForContiguousMemory(
1843     IN PVOID BaseAddress,
1844     IN PFN_NUMBER BaseAddressPages,
1845     IN PFN_NUMBER SizeInPages,
1846     IN PFN_NUMBER LowestPfn,
1847     IN PFN_NUMBER HighestPfn,
1848     IN PFN_NUMBER BoundaryPfn,
1849     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1850 );
1851
1852 PMDL
1853 NTAPI
1854 MiAllocatePagesForMdl(
1855     IN PHYSICAL_ADDRESS LowAddress,
1856     IN PHYSICAL_ADDRESS HighAddress,
1857     IN PHYSICAL_ADDRESS SkipBytes,
1858     IN SIZE_T TotalBytes,
1859     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1860     IN ULONG Flags
1861 );
1862
1863 PVOID
1864 NTAPI
1865 MiMapLockedPagesInUserSpace(
1866     IN PMDL Mdl,
1867     IN PVOID BaseVa,
1868     IN MEMORY_CACHING_TYPE CacheType,
1869     IN PVOID BaseAddress
1870 );
1871
1872 VOID
1873 NTAPI
1874 MiUnmapLockedPagesInUserSpace(
1875     IN PVOID BaseAddress,
1876     IN PMDL Mdl
1877 );
1878
1879 VOID
1880 NTAPI
1881 MiInsertPageInList(
1882     IN PMMPFNLIST ListHead,
1883     IN PFN_NUMBER PageFrameIndex
1884 );
1885
1886 VOID
1887 NTAPI
1888 MiUnlinkFreeOrZeroedPage(
1889     IN PMMPFN Entry
1890 );
1891
1892 VOID
1893 NTAPI
1894 MiUnlinkPageFromList(
1895     IN PMMPFN Pfn
1896 );
1897
1898 PFN_NUMBER
1899 NTAPI
1900 MiAllocatePfn(
1901     IN PMMPTE PointerPte,
1902     IN ULONG Protection
1903 );
1904
1905 VOID
1906 NTAPI
1907 MiInitializePfn(
1908     IN PFN_NUMBER PageFrameIndex,
1909     IN PMMPTE PointerPte,
1910     IN BOOLEAN Modified
1911 );
1912
1913 NTSTATUS
1914 NTAPI
1915 MiInitializeAndChargePfn(
1916     OUT PPFN_NUMBER PageFrameIndex,
1917     IN PMMPTE PointerPde,
1918     IN PFN_NUMBER ContainingPageFrame,
1919     IN BOOLEAN SessionAllocation
1920 );
1921
1922 VOID
1923 NTAPI
1924 MiInitializePfnAndMakePteValid(
1925     IN PFN_NUMBER PageFrameIndex,
1926     IN PMMPTE PointerPte,
1927     IN MMPTE TempPte
1928 );
1929
1930 VOID
1931 NTAPI
1932 MiInitializePfnForOtherProcess(
1933     IN PFN_NUMBER PageFrameIndex,
1934     IN PMMPTE PointerPte,
1935     IN PFN_NUMBER PteFrame
1936 );
1937
1938 VOID
1939 NTAPI
1940 MiDecrementShareCount(
1941     IN PMMPFN Pfn1,
1942     IN PFN_NUMBER PageFrameIndex
1943 );
1944
1945 PFN_NUMBER
1946 NTAPI
1947 MiRemoveAnyPage(
1948     IN ULONG Color
1949 );
1950
1951 PFN_NUMBER
1952 NTAPI
1953 MiRemoveZeroPage(
1954     IN ULONG Color
1955 );
1956
1957 VOID
1958 NTAPI
1959 MiZeroPhysicalPage(
1960     IN PFN_NUMBER PageFrameIndex
1961 );
1962
1963 VOID
1964 NTAPI
1965 MiInsertPageInFreeList(
1966     IN PFN_NUMBER PageFrameIndex
1967 );
1968
1969 PFN_COUNT
1970 NTAPI
1971 MiDeleteSystemPageableVm(
1972     IN PMMPTE PointerPte,
1973     IN PFN_NUMBER PageCount,
1974     IN ULONG Flags,
1975     OUT PPFN_NUMBER ValidPages
1976 );
1977
1978 ULONG
1979 NTAPI
1980 MiGetPageProtection(
1981     IN PMMPTE PointerPte
1982 );
1983
1984 PLDR_DATA_TABLE_ENTRY
1985 NTAPI
1986 MiLookupDataTableEntry(
1987     IN PVOID Address
1988 );
1989
1990 VOID
1991 NTAPI
1992 MiInitializeDriverLargePageList(
1993     VOID
1994 );
1995
1996 VOID
1997 NTAPI
1998 MiInitializeLargePageSupport(
1999     VOID
2000 );
2001
2002 VOID
2003 NTAPI
2004 MiSyncCachedRanges(
2005     VOID
2006 );
2007
2008 BOOLEAN
2009 NTAPI
2010 MiIsPfnInUse(
2011     IN PMMPFN Pfn1
2012 );
2013
2014 PMMVAD
2015 NTAPI
2016 MiLocateAddress(
2017     IN PVOID VirtualAddress
2018 );
2019
2020 PMMADDRESS_NODE
2021 NTAPI
2022 MiCheckForConflictingNode(
2023     IN ULONG_PTR StartVpn,
2024     IN ULONG_PTR EndVpn,
2025     IN PMM_AVL_TABLE Table
2026 );
2027
2028 TABLE_SEARCH_RESULT
2029 NTAPI
2030 MiFindEmptyAddressRangeDownTree(
2031     IN SIZE_T Length,
2032     IN ULONG_PTR BoundaryAddress,
2033     IN ULONG_PTR Alignment,
2034     IN PMM_AVL_TABLE Table,
2035     OUT PULONG_PTR Base,
2036     OUT PMMADDRESS_NODE *Parent
2037 );
2038
2039 NTSTATUS
2040 NTAPI
2041 MiFindEmptyAddressRangeDownBasedTree(
2042     IN SIZE_T Length,
2043     IN ULONG_PTR BoundaryAddress,
2044     IN ULONG_PTR Alignment,
2045     IN PMM_AVL_TABLE Table,
2046     OUT PULONG_PTR Base
2047 );
2048
2049 NTSTATUS
2050 NTAPI
2051 MiFindEmptyAddressRangeInTree(
2052     IN SIZE_T Length,
2053     IN ULONG_PTR Alignment,
2054     IN PMM_AVL_TABLE Table,
2055     OUT PMMADDRESS_NODE *PreviousVad,
2056     OUT PULONG_PTR Base
2057 );
2058
2059 NTSTATUS
2060 NTAPI
2061 MiCheckSecuredVad(
2062     IN PMMVAD Vad,
2063     IN PVOID Base,
2064     IN SIZE_T Size,
2065     IN ULONG ProtectionMask
2066 );
2067
2068 VOID
2069 NTAPI
2070 MiInsertVad(
2071     IN PMMVAD Vad,
2072     IN PEPROCESS Process
2073 );
2074
2075 VOID
2076 NTAPI
2077 MiInsertBasedSection(
2078     IN PSECTION Section
2079 );
2080
2081 NTSTATUS
2082 NTAPI
2083 MiUnmapViewOfSection(
2084     IN PEPROCESS Process,
2085     IN PVOID BaseAddress,
2086     IN ULONG Flags
2087 );
2088
2089 NTSTATUS
2090 NTAPI
2091 MiRosUnmapViewOfSection(
2092     IN PEPROCESS Process,
2093     IN PVOID BaseAddress,
2094     IN ULONG Flags
2095 );
2096
2097 VOID
2098 NTAPI
2099 MiInsertNode(
2100     IN PMM_AVL_TABLE Table,
2101     IN PMMADDRESS_NODE NewNode,
2102     PMMADDRESS_NODE Parent,
2103     TABLE_SEARCH_RESULT Result
2104 );
2105
2106 VOID
2107 NTAPI
2108 MiRemoveNode(
2109     IN PMMADDRESS_NODE Node,
2110     IN PMM_AVL_TABLE Table
2111 );
2112
2113 PMMADDRESS_NODE
2114 NTAPI
2115 MiGetPreviousNode(
2116     IN PMMADDRESS_NODE Node
2117 );
2118
2119 PMMADDRESS_NODE
2120 NTAPI
2121 MiGetNextNode(
2122     IN PMMADDRESS_NODE Node
2123 );
2124
2125 BOOLEAN
2126 NTAPI
2127 MiInitializeSystemSpaceMap(
2128     IN PMMSESSION InputSession OPTIONAL
2129 );
2130
2131 VOID
2132 NTAPI
2133 MiSessionRemoveProcess(
2134     VOID
2135 );
2136
2137 VOID
2138 NTAPI
2139 MiReleaseProcessReferenceToSessionDataPage(
2140     IN PMM_SESSION_SPACE SessionGlobal
2141 );
2142
2143 VOID
2144 NTAPI
2145 MiSessionAddProcess(
2146     IN PEPROCESS NewProcess
2147 );
2148
2149 NTSTATUS
2150 NTAPI
2151 MiSessionCommitPageTables(
2152     IN PVOID StartVa,
2153     IN PVOID EndVa
2154 );
2155
2156 ULONG
2157 NTAPI
2158 MiMakeProtectionMask(
2159     IN ULONG Protect
2160 );
2161
2162 VOID
2163 NTAPI
2164 MiDeleteVirtualAddresses(
2165     IN ULONG_PTR Va,
2166     IN ULONG_PTR EndingAddress,
2167     IN PMMVAD Vad
2168 );
2169
2170 ULONG
2171 NTAPI
2172 MiMakeSystemAddressValid(
2173     IN PVOID PageTableVirtualAddress,
2174     IN PEPROCESS CurrentProcess
2175 );
2176
2177 ULONG
2178 NTAPI
2179 MiMakeSystemAddressValidPfn(
2180     IN PVOID VirtualAddress,
2181     IN KIRQL OldIrql
2182 );
2183
2184 VOID
2185 NTAPI
2186 MiRemoveMappedView(
2187     IN PEPROCESS CurrentProcess,
2188     IN PMMVAD Vad
2189 );
2190
2191 PSUBSECTION
2192 NTAPI
2193 MiLocateSubsection(
2194     IN PMMVAD Vad,
2195     IN ULONG_PTR Vpn
2196 );
2197
2198 NTSTATUS
2199 NTAPI
2200 MiQueryMemorySectionName(
2201     IN HANDLE ProcessHandle,
2202     IN PVOID BaseAddress,
2203     OUT PVOID MemoryInformation,
2204     IN SIZE_T MemoryInformationLength,
2205     OUT PSIZE_T ReturnLength
2206 );
2207
2208 NTSTATUS
2209 NTAPI
2210 MiRosUnmapViewInSystemSpace(
2211     IN PVOID MappedBase
2212 );
2213
2214 POOL_TYPE
2215 NTAPI
2216 MmDeterminePoolType(
2217     IN PVOID PoolAddress
2218 );
2219
2220 VOID
2221 NTAPI
2222 MiMakePdeExistAndMakeValid(
2223     IN PMMPTE PointerPde,
2224     IN PEPROCESS TargetProcess,
2225     IN KIRQL OldIrql
2226 );
2227
2228 //
2229 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2230 // free pages are available. In some scenarios, we don't/can't run that piece of
2231 // code and would rather only have a real zero page. If we can't have a zero page,
2232 // then we'd like to have our own code to grab a free page and zero it out, by
2233 // using MiRemoveAnyPage. This macro implements this.
2234 //
2235 PFN_NUMBER
2236 FORCEINLINE
2237 MiRemoveZeroPageSafe(IN ULONG Color)
2238 {
2239     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2240     return 0;
2241 }
2242
2243 /* EOF */