ntoskrnl/mm/ARM3/miarm.h

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/miarm.h
   5  * PURPOSE:         ARM Memory Manager Header
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 #ifndef _M_AMD64
  10
  11 #define MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING   ((255 * _1MB) >> PAGE_SHIFT)
  12 #define MI_MIN_PAGES_FOR_SYSPTE_TUNING          ((19 * _1MB) >> PAGE_SHIFT)
  13 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST           ((32 * _1MB) >> PAGE_SHIFT)
  14 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST_BOOST     ((256 * _1MB) >> PAGE_SHIFT)
  15 #define MI_MAX_INIT_NONPAGED_POOL_SIZE          (128 * _1MB)
  16 #define MI_MAX_NONPAGED_POOL_SIZE               (128 * _1MB)
  17 #define MI_MAX_FREE_PAGE_LISTS                  4
  18
  19 #define MI_MIN_INIT_PAGED_POOLSIZE              (32 * _1MB)
  20
  21 #define MI_SESSION_VIEW_SIZE                    (48 * _1MB)
  22 #define MI_SESSION_POOL_SIZE                    (16 * _1MB)
  23 #define MI_SESSION_IMAGE_SIZE                   (8 * _1MB)
  24 #define MI_SESSION_WORKING_SET_SIZE             (4 * _1MB)
  25 #define MI_SESSION_SIZE                         (MI_SESSION_VIEW_SIZE + \
  26                                                  MI_SESSION_POOL_SIZE + \
  27                                                  MI_SESSION_IMAGE_SIZE + \
  28                                                  MI_SESSION_WORKING_SET_SIZE)
  29
  30 #define MI_SYSTEM_VIEW_SIZE                     (32 * _1MB)
  31
  32 #define MI_HIGHEST_USER_ADDRESS                 (PVOID)0x7FFEFFFF
  33 #define MI_USER_PROBE_ADDRESS                   (PVOID)0x7FFF0000
  34 #define MI_DEFAULT_SYSTEM_RANGE_START           (PVOID)0x80000000
  35 #define MI_SYSTEM_CACHE_WS_START                (PVOID)0xC0C00000
  36 #define MI_PAGED_POOL_START                     (PVOID)0xE1000000
  37 #define MI_NONPAGED_POOL_END                    (PVOID)0xFFBE0000
  38 #define MI_DEBUG_MAPPING                        (PVOID)0xFFBFF000
  39
  40 #define MI_SYSTEM_PTE_BASE                      (PVOID)MiAddressToPte(NULL)
  41
  42 #define MI_MIN_SECONDARY_COLORS                 8
  43 #define MI_SECONDARY_COLORS                     64
  44 #define MI_MAX_SECONDARY_COLORS                 1024
  45
  46 #define MI_MIN_ALLOCATION_FRAGMENT              (4 * _1KB)
  47 #define MI_ALLOCATION_FRAGMENT                  (64 * _1KB)
  48 #define MI_MAX_ALLOCATION_FRAGMENT              (2  * _1MB)
  49
  50 #define MM_HIGHEST_VAD_ADDRESS \
  51     (PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (16 * PAGE_SIZE))
  52 #define MI_LOWEST_VAD_ADDRESS                   (PVOID)MM_LOWEST_USER_ADDRESS
  53
  54 #define MI_DEFAULT_SYSTEM_PTE_COUNT             50000
  55 #define MI_MAX_ZERO_BITS                        21
  56
  57 #endif /* !_M_AMD64 */
  58
  59 /* Make the code cleaner with some definitions for size multiples */
  60 #define _1KB (1024u)
  61 #define _1MB (1024 * _1KB)
  62 #define _1GB (1024 * _1MB)
  63
  64 /* Everyone loves 64K */
  65 #define _64K (64 * _1KB)
  66
  67 /* Area mapped by a PDE */
  68 #define PDE_MAPPED_VA  (PTE_COUNT * PAGE_SIZE)
  69
  70 /* Size of a page table */
  71 #define PT_SIZE  (PTE_COUNT * sizeof(MMPTE))
  72
  73 /* Size of a page directory */
  74 #define PD_SIZE  (PDE_COUNT * sizeof(MMPDE))
  75
  76 /* Size of all page directories for a process */
  77 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
  78
  79 /* Architecture specific count of PDEs in a directory, and count of PTEs in a PT */
  80 #ifdef _M_IX86
  81 #define PD_COUNT  1
  82 #define PDE_COUNT 1024
  83 #define PTE_COUNT 1024
  84 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE);
  85 #define MiIsPteOnPdeBoundary(PointerPte) \
  86     ((((ULONG_PTR)PointerPte) & (PAGE_SIZE - 1)) == 0)
  87 #elif _M_ARM
  88 #define PD_COUNT  1
  89 #define PDE_COUNT 4096
  90 #define PTE_COUNT 256
  91 #else
  92 #define PD_COUNT  PPE_PER_PAGE
  93 #define PDE_COUNT PDE_PER_PAGE
  94 #define PTE_COUNT PTE_PER_PAGE
  95 #endif
  96
  97 //
  98 // Protection Bits part of the internal memory manager Protection Mask, from:
  99 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
 100 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
 101 // and public assertions.
 102 //
 103 #define MM_ZERO_ACCESS         0
 104 #define MM_READONLY            1
 105 #define MM_EXECUTE             2
 106 #define MM_EXECUTE_READ        3
 107 #define MM_READWRITE           4
 108 #define MM_WRITECOPY           5
 109 #define MM_EXECUTE_READWRITE   6
 110 #define MM_EXECUTE_WRITECOPY   7
 111 #define MM_PROTECT_ACCESS      7
 112
 113 //
 114 // These are flags on top of the actual protection mask
 115 //
 116 #define MM_NOCACHE            0x08
 117 #define MM_GUARDPAGE          0x10
 118 #define MM_WRITECOMBINE       0x18
 119 #define MM_PROTECT_SPECIAL    0x18
 120
 121 //
 122 // These are special cases
 123 //
 124 #define MM_DECOMMIT           (MM_ZERO_ACCESS | MM_GUARDPAGE)
 125 #define MM_NOACCESS           (MM_ZERO_ACCESS | MM_WRITECOMBINE)
 126 #define MM_OUTSWAPPED_KSTACK  (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
 127 #define MM_INVALID_PROTECTION  0xFFFFFFFF
 128
 129 //
 130 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
 131 // The Memory Manager's definition define the attributes that must be preserved
 132 // and these PTE definitions describe the attributes in the hardware sense. This
 133 // helps deal with hardware differences between the actual boolean expression of
 134 // the argument.
 135 //
 136 // For example, in the logical attributes, we want to express read-only as a flag
 137 // but on x86, it is writability that must be set. On the other hand, on x86, just
 138 // like in the kernel, it is disabling the caches that requires a special flag,
 139 // while on certain architectures such as ARM, it is enabling the cache which
 140 // requires a flag.
 141 //
 142 #if defined(_M_IX86) || defined(_M_AMD64)
 143 //
 144 // Access Flags
 145 //
 146 #define PTE_READONLY            0 // Doesn't exist on x86
 147 #define PTE_EXECUTE             0 // Not worrying about NX yet
 148 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 149 #define PTE_READWRITE           0x2
 150 #define PTE_WRITECOPY           0x200
 151 #define PTE_EXECUTE_READWRITE   0x2 // Not worrying about NX yet
 152 #define PTE_EXECUTE_WRITECOPY   0x200
 153 #define PTE_PROTOTYPE           0x400
 154
 155 //
 156 // State Flags
 157 //
 158 #define PTE_VALID               0x1
 159 #define PTE_ACCESSED            0x20
 160 #define PTE_DIRTY               0x40
 161
 162 //
 163 // Cache flags
 164 //
 165 #define PTE_ENABLE_CACHE        0
 166 #define PTE_DISABLE_CACHE       0x10
 167 #define PTE_WRITECOMBINED_CACHE 0x10
 168 #elif defined(_M_ARM)
 169 #define PTE_READONLY            0x200
 170 #define PTE_EXECUTE             0 // Not worrying about NX yet
 171 #define PTE_EXECUTE_READ        0 // Not worrying about NX yet
 172 #define PTE_READWRITE           0 // Doesn't exist on ARM
 173 #define PTE_WRITECOPY           0 // Doesn't exist on ARM
 174 #define PTE_EXECUTE_READWRITE   0 // Not worrying about NX yet
 175 #define PTE_EXECUTE_WRITECOPY   0 // Not worrying about NX yet
 176 #define PTE_PROTOTYPE           0x400 // Using the Shared bit
 177 //
 178 // Cache flags
 179 //
 180 #define PTE_ENABLE_CACHE        0
 181 #define PTE_DISABLE_CACHE       0x10
 182 #define PTE_WRITECOMBINED_CACHE 0x10
 183 #else
 184 #error Define these please!
 185 #endif
 186
 187 extern const ULONG_PTR MmProtectToPteMask[32];
 188 extern const ULONG MmProtectToValue[32];
 189
 190 //
 191 // Assertions for session images, addresses, and PTEs
 192 //
 193 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
 194     (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
 195
 196 #define MI_IS_SESSION_ADDRESS(Address) \
 197     (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
 198
 199 #define MI_IS_SESSION_PTE(Pte) \
 200     ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
 201
 202 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
 203     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
 204
 205 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
 206     (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
 207
 208 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
 209     (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
 210
 211 //
 212 // Corresponds to MMPTE_SOFTWARE.Protection
 213 //
 214 #ifdef _M_IX86
 215 #define MM_PTE_SOFTWARE_PROTECTION_BITS   5
 216 #elif _M_ARM
 217 #define MM_PTE_SOFTWARE_PROTECTION_BITS   6
 218 #elif _M_AMD64
 219 #define MM_PTE_SOFTWARE_PROTECTION_BITS   5
 220 #else
 221 #error Define these please!
 222 #endif
 223
 224 //
 225 // Creates a software PTE with the given protection
 226 //
 227 #define MI_MAKE_SOFTWARE_PTE(p, x)          ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
 228
 229 //
 230 // Marks a PTE as deleted
 231 //
 232 #define MI_SET_PFN_DELETED(x)               ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
 233 #define MI_IS_PFN_DELETED(x)                ((ULONG_PTR)((x)->PteAddress) & 1)
 234
 235 //
 236 // Special values for LoadedImports
 237 //
 238 #define MM_SYSLDR_NO_IMPORTS   (PVOID)0xFFFFFFFE
 239 #define MM_SYSLDR_BOOT_LOADED  (PVOID)0xFFFFFFFF
 240 #define MM_SYSLDR_SINGLE_ENTRY 0x1
 241
 242 //
 243 // Number of initial session IDs
 244 //
 245 #define MI_INITIAL_SESSION_IDS  64
 246
 247 #if defined(_M_IX86) || defined(_M_ARM)
 248 //
 249 // PFN List Sentinel
 250 //
 251 #define LIST_HEAD 0xFFFFFFFF
 252
 253 //
 254 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
 255 // we need a manual definition suited to the number of bits in the PteFrame.
 256 // This is used as a LIST_HEAD for the colored list
 257 //
 258 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
 259 #elif defined(_M_AMD64)
 260 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
 261 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
 262 #else
 263 #error Define these please!
 264 #endif
 265
 266 //
 267 // Special IRQL value (found in assertions)
 268 //
 269 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 270
 271 //
 272 // Returns the color of a page
 273 //
 274 #define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
 275 #define MI_GET_NEXT_COLOR()                 (MI_GET_PAGE_COLOR(++MmSystemPageColor))
 276 #define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
 277
 278 #ifndef _M_AMD64
 279 //
 280 // Decodes a Prototype PTE into the underlying PTE
 281 //
 282 #define MiProtoPteToPte(x)                  \
 283     (PMMPTE)((ULONG_PTR)MmPagedPoolStart +  \
 284              (((x)->u.Proto.ProtoAddressHigh << 9) | (x)->u.Proto.ProtoAddressLow << 2))
 285
 286 //
 287 // Decodes a Prototype PTE into the underlying PTE
 288 //
 289 #define MiSubsectionPteToSubsection(x)                              \
 290     ((x)->u.Subsect.WhichPool == PagedPool) ?                       \
 291         (PMMPTE)((ULONG_PTR)MmSubsectionBase +                      \
 292                  (((x)->u.Subsect.SubsectionAddressHigh << 7) |     \
 293                    (x)->u.Subsect.SubsectionAddressLow << 3)) :     \
 294         (PMMPTE)((ULONG_PTR)MmNonPagedPoolEnd -                     \
 295                 (((x)->u.Subsect.SubsectionAddressHigh << 7) |      \
 296                   (x)->u.Subsect.SubsectionAddressLow << 3))
 297 #endif
 298
 299 //
 300 // Prototype PTEs that don't yet have a pagefile association
 301 //
 302 #ifdef _M_AMD64
 303 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
 304 #else
 305 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
 306 #endif
 307
 308 //
 309 // Number of session lists in the MM_SESSIONS_SPACE structure
 310 //
 311 #if defined(_M_AMD64)
 312 #define SESSION_POOL_LOOKASIDES 21
 313 #elif defined(_M_IX86)
 314 #define SESSION_POOL_LOOKASIDES 26
 315 #else
 316 #error Not Defined!
 317 #endif
 318
 319 //
 320 // Number of session data and tag pages
 321 //
 322 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 323 #define MI_SESSION_TAG_PAGES_MAXIMUM  (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
 324
 325 //
 326 // Used by MiCheckSecuredVad
 327 //
 328 #define MM_READ_WRITE_ALLOWED   11
 329 #define MM_READ_ONLY_ALLOWED    10
 330 #define MM_NO_ACCESS_ALLOWED    01
 331 #define MM_DELETE_CHECK         85
 332
 333 //
 334 // System views are binned into 64K chunks
 335 //
 336 #define MI_SYSTEM_VIEW_BUCKET_SIZE  _64K
 337
 338 //
 339 // FIXFIX: These should go in ex.h after the pool merge
 340 //
 341 #ifdef _M_AMD64
 342 #define POOL_BLOCK_SIZE 16
 343 #else
 344 #define POOL_BLOCK_SIZE  8
 345 #endif
 346 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
 347 #define BASE_POOL_TYPE_MASK 1
 348 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
 349
 350 //
 351 // Pool debugging/analysis/tracing flags
 352 //
 353 #define POOL_FLAG_CHECK_TIMERS 0x1
 354 #define POOL_FLAG_CHECK_WORKERS 0x2
 355 #define POOL_FLAG_CHECK_RESOURCES 0x4
 356 #define POOL_FLAG_VERIFIER 0x8
 357 #define POOL_FLAG_CHECK_DEADLOCK 0x10
 358 #define POOL_FLAG_SPECIAL_POOL 0x20
 359 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
 360 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
 361
 362 //
 363 // BAD_POOL_HEADER codes during pool bugcheck
 364 //
 365 #define POOL_CORRUPTED_LIST 3
 366 #define POOL_SIZE_OR_INDEX_MISMATCH 5
 367 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
 368 #define POOL_HEADER_NOT_ALIGNED 7
 369 #define POOL_HEADER_IS_ZERO 8
 370 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
 371 #define POOL_ENTRY_NOT_FOUND 10
 372
 373 //
 374 // BAD_POOL_CALLER codes during pool bugcheck
 375 //
 376 #define POOL_ENTRY_CORRUPTED 1
 377 #define POOL_ENTRY_ALREADY_FREE 6
 378 #define POOL_ENTRY_NOT_ALLOCATED 7
 379 #define POOL_ALLOC_IRQL_INVALID 8
 380 #define POOL_FREE_IRQL_INVALID 9
 381 #define POOL_BILLED_PROCESS_INVALID 13
 382 #define POOL_HEADER_SIZE_INVALID 32
 383
 384 typedef struct _POOL_DESCRIPTOR
 385 {
 386     POOL_TYPE PoolType;
 387     ULONG PoolIndex;
 388     ULONG RunningAllocs;
 389     ULONG RunningDeAllocs;
 390     ULONG TotalPages;
 391     ULONG TotalBigPages;
 392     ULONG Threshold;
 393     PVOID LockAddress;
 394     PVOID PendingFrees;
 395     LONG PendingFreeDepth;
 396     SIZE_T TotalBytes;
 397     SIZE_T Spare0;
 398     LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE];
 399 } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR;
 400
 401 typedef struct _POOL_HEADER
 402 {
 403     union
 404     {
 405         struct
 406         {
 407 #ifdef _M_AMD64
 408             USHORT PreviousSize:8;
 409             USHORT PoolIndex:8;
 410             USHORT BlockSize:8;
 411             USHORT PoolType:8;
 412 #else
 413             USHORT PreviousSize:9;
 414             USHORT PoolIndex:7;
 415             USHORT BlockSize:9;
 416             USHORT PoolType:7;
 417 #endif
 418         };
 419         ULONG Ulong1;
 420     };
 421 #ifdef _M_AMD64
 422     ULONG PoolTag;
 423 #endif
 424     union
 425     {
 426 #ifdef _M_AMD64
 427         PEPROCESS ProcessBilled;
 428 #else
 429         ULONG PoolTag;
 430 #endif
 431         struct
 432         {
 433             USHORT AllocatorBackTraceIndex;
 434             USHORT PoolTagHash;
 435         };
 436     };
 437 } POOL_HEADER, *PPOOL_HEADER;
 438
 439 C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE);
 440 C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY));
 441
 442 typedef struct _POOL_TRACKER_TABLE
 443 {
 444     ULONG Key;
 445     LONG NonPagedAllocs;
 446     LONG NonPagedFrees;
 447     SIZE_T NonPagedBytes;
 448     LONG PagedAllocs;
 449     LONG PagedFrees;
 450     SIZE_T PagedBytes;
 451 } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE;
 452
 453 typedef struct _POOL_TRACKER_BIG_PAGES
 454 {
 455     PVOID Va;
 456     ULONG Key;
 457     ULONG NumberOfPages;
 458     PVOID QuotaObject;
 459 } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES;
 460
 461 extern ULONG ExpNumberOfPagedPools;
 462 extern POOL_DESCRIPTOR NonPagedPoolDescriptor;
 463 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1];
 464 extern PPOOL_TRACKER_TABLE PoolTrackTable;
 465
 466 //
 467 // END FIXFIX
 468 //
 469
 470 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
 471 {
 472     LIST_ENTRY Links;
 473     UNICODE_STRING BaseName;
 474 } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY;
 475
 476 typedef enum _MMSYSTEM_PTE_POOL_TYPE
 477 {
 478     SystemPteSpace,
 479     NonPagedPoolExpansion,
 480     MaximumPtePoolTypes
 481 } MMSYSTEM_PTE_POOL_TYPE;
 482
 483 typedef enum _MI_PFN_CACHE_ATTRIBUTE
 484 {
 485     MiNonCached,
 486     MiCached,
 487     MiWriteCombined,
 488     MiNotMapped
 489 } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE;
 490
 491 typedef struct _PHYSICAL_MEMORY_RUN
 492 {
 493     PFN_NUMBER BasePage;
 494     PFN_NUMBER PageCount;
 495 } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN;
 496
 497 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
 498 {
 499     ULONG NumberOfRuns;
 500     PFN_NUMBER NumberOfPages;
 501     PHYSICAL_MEMORY_RUN Run[1];
 502 } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR;
 503
 504 typedef struct _MMCOLOR_TABLES
 505 {
 506     PFN_NUMBER Flink;
 507     PVOID Blink;
 508     PFN_NUMBER Count;
 509 } MMCOLOR_TABLES, *PMMCOLOR_TABLES;
 510
 511 typedef struct _MI_LARGE_PAGE_RANGES
 512 {
 513     PFN_NUMBER StartFrame;
 514     PFN_NUMBER LastFrame;
 515 } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES;
 516
 517 typedef struct _MMVIEW
 518 {
 519     ULONG_PTR Entry;
 520     PCONTROL_AREA ControlArea;
 521 } MMVIEW, *PMMVIEW;
 522
 523 typedef struct _MMSESSION
 524 {
 525     KGUARDED_MUTEX SystemSpaceViewLock;
 526     PKGUARDED_MUTEX SystemSpaceViewLockPointer;
 527     PCHAR SystemSpaceViewStart;
 528     PMMVIEW SystemSpaceViewTable;
 529     ULONG SystemSpaceHashSize;
 530     ULONG SystemSpaceHashEntries;
 531     ULONG SystemSpaceHashKey;
 532     ULONG BitmapFailures;
 533     PRTL_BITMAP SystemSpaceBitMap;
 534 } MMSESSION, *PMMSESSION;
 535
 536 typedef struct _MM_SESSION_SPACE_FLAGS
 537 {
 538     ULONG Initialized:1;
 539     ULONG DeletePending:1;
 540     ULONG Filler:30;
 541 } MM_SESSION_SPACE_FLAGS;
 542
 543 typedef struct _MM_SESSION_SPACE
 544 {
 545     struct _MM_SESSION_SPACE *GlobalVirtualAddress;
 546     LONG ReferenceCount;
 547     union
 548     {
 549         ULONG LongFlags;
 550         MM_SESSION_SPACE_FLAGS Flags;
 551     } u;
 552     ULONG SessionId;
 553     LIST_ENTRY ProcessList;
 554     LARGE_INTEGER LastProcessSwappedOutTime;
 555     PFN_NUMBER SessionPageDirectoryIndex;
 556     SIZE_T NonPageablePages;
 557     SIZE_T CommittedPages;
 558     PVOID PagedPoolStart;
 559     PVOID PagedPoolEnd;
 560     PMMPTE PagedPoolBasePde;
 561     ULONG Color;
 562     LONG ResidentProcessCount;
 563     ULONG SessionPoolAllocationFailures[4];
 564     LIST_ENTRY ImageList;
 565     LCID LocaleId;
 566     ULONG AttachCount;
 567     KEVENT AttachEvent;
 568     PEPROCESS LastProcess;
 569     LONG ProcessReferenceToSession;
 570     LIST_ENTRY WsListEntry;
 571     GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES];
 572     MMSESSION Session;
 573     KGUARDED_MUTEX PagedPoolMutex;
 574     MM_PAGED_POOL_INFO PagedPoolInfo;
 575     MMSUPPORT Vm;
 576     PMMWSLE Wsle;
 577     PDRIVER_UNLOAD Win32KDriverUnload;
 578     POOL_DESCRIPTOR PagedPool;
 579 #if defined (_M_AMD64)
 580     MMPTE PageDirectory;
 581 #else
 582     PMMPTE PageTables;
 583 #endif
 584 #if defined (_M_AMD64)
 585     PMMPTE SpecialPoolFirstPte;
 586     PMMPTE SpecialPoolLastPte;
 587     PMMPTE NextPdeForSpecialPoolExpansion;
 588     PMMPTE LastPdeForSpecialPoolExpansion;
 589     PFN_NUMBER SpecialPagesInUse;
 590 #endif
 591     LONG ImageLoadingCount;
 592 } MM_SESSION_SPACE, *PMM_SESSION_SPACE;
 593
 594 extern PMM_SESSION_SPACE MmSessionSpace;
 595 extern MMPTE HyperTemplatePte;
 596 extern MMPDE ValidKernelPde;
 597 extern MMPTE ValidKernelPte;
 598 extern MMPDE ValidKernelPdeLocal;
 599 extern MMPTE ValidKernelPteLocal;
 600 extern MMPDE DemandZeroPde;
 601 extern MMPTE DemandZeroPte;
 602 extern MMPTE PrototypePte;
 603 extern MMPTE MmDecommittedPte;
 604 extern BOOLEAN MmLargeSystemCache;
 605 extern BOOLEAN MmZeroPageFile;
 606 extern BOOLEAN MmProtectFreedNonPagedPool;
 607 extern BOOLEAN MmTrackLockedPages;
 608 extern BOOLEAN MmTrackPtes;
 609 extern BOOLEAN MmDynamicPfn;
 610 extern BOOLEAN MmMirroring;
 611 extern BOOLEAN MmMakeLowMemory;
 612 extern BOOLEAN MmEnforceWriteProtection;
 613 extern SIZE_T MmAllocationFragment;
 614 extern ULONG MmConsumedPoolPercentage;
 615 extern ULONG MmVerifyDriverBufferType;
 616 extern ULONG MmVerifyDriverLevel;
 617 extern WCHAR MmVerifyDriverBuffer[512];
 618 extern WCHAR MmLargePageDriverBuffer[512];
 619 extern LIST_ENTRY MiLargePageDriverList;
 620 extern BOOLEAN MiLargePageAllDrivers;
 621 extern ULONG MmVerifyDriverBufferLength;
 622 extern ULONG MmLargePageDriverBufferLength;
 623 extern SIZE_T MmSizeOfNonPagedPoolInBytes;
 624 extern SIZE_T MmMaximumNonPagedPoolInBytes;
 625 extern PFN_NUMBER MmMaximumNonPagedPoolInPages;
 626 extern PFN_NUMBER MmSizeOfPagedPoolInPages;
 627 extern PVOID MmNonPagedSystemStart;
 628 extern SIZE_T MiNonPagedSystemSize;
 629 extern PVOID MmNonPagedPoolStart;
 630 extern PVOID MmNonPagedPoolExpansionStart;
 631 extern PVOID MmNonPagedPoolEnd;
 632 extern SIZE_T MmSizeOfPagedPoolInBytes;
 633 extern PVOID MmPagedPoolStart;
 634 extern PVOID MmPagedPoolEnd;
 635 extern PVOID MmSessionBase;
 636 extern SIZE_T MmSessionSize;
 637 extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte;
 638 extern PMMPTE MiFirstReservedZeroingPte;
 639 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType];
 640 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
 641 extern SIZE_T MmBootImageSize;
 642 extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes];
 643 extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes];
 644 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
 645 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
 646 extern ULONG_PTR MxPfnAllocation;
 647 extern MM_PAGED_POOL_INFO MmPagedPoolInfo;
 648 extern RTL_BITMAP MiPfnBitMap;
 649 extern KGUARDED_MUTEX MmPagedPoolMutex;
 650 extern KGUARDED_MUTEX MmSectionCommitMutex;
 651 extern PVOID MmPagedPoolStart;
 652 extern PVOID MmPagedPoolEnd;
 653 extern PVOID MmNonPagedSystemStart;
 654 extern PVOID MiSystemViewStart;
 655 extern SIZE_T MmSystemViewSize;
 656 extern PVOID MmSessionBase;
 657 extern PVOID MiSessionSpaceEnd;
 658 extern PMMPTE MiSessionImagePteStart;
 659 extern PMMPTE MiSessionImagePteEnd;
 660 extern PMMPTE MiSessionBasePte;
 661 extern PMMPTE MiSessionLastPte;
 662 extern SIZE_T MmSizeOfPagedPoolInBytes;
 663 extern PMMPDE MmSystemPagePtes;
 664 extern PVOID MmSystemCacheStart;
 665 extern PVOID MmSystemCacheEnd;
 666 extern MMSUPPORT MmSystemCacheWs;
 667 extern SIZE_T MmAllocatedNonPagedPool;
 668 extern ULONG MmSpecialPoolTag;
 669 extern PVOID MmHyperSpaceEnd;
 670 extern PMMWSL MmSystemCacheWorkingSetList;
 671 extern SIZE_T MmMinimumNonPagedPoolSize;
 672 extern ULONG MmMinAdditionNonPagedPoolPerMb;
 673 extern SIZE_T MmDefaultMaximumNonPagedPool;
 674 extern ULONG MmMaxAdditionNonPagedPoolPerMb;
 675 extern ULONG MmSecondaryColors;
 676 extern ULONG MmSecondaryColorMask;
 677 extern ULONG MmNumberOfSystemPtes;
 678 extern ULONG MmMaximumNonPagedPoolPercent;
 679 extern ULONG MmLargeStackSize;
 680 extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1];
 681 extern MMPFNLIST MmStandbyPageListByPriority[8];
 682 extern ULONG MmProductType;
 683 extern MM_SYSTEMSIZE MmSystemSize;
 684 extern PKEVENT MiLowMemoryEvent;
 685 extern PKEVENT MiHighMemoryEvent;
 686 extern PKEVENT MiLowPagedPoolEvent;
 687 extern PKEVENT MiHighPagedPoolEvent;
 688 extern PKEVENT MiLowNonPagedPoolEvent;
 689 extern PKEVENT MiHighNonPagedPoolEvent;
 690 extern PFN_NUMBER MmLowMemoryThreshold;
 691 extern PFN_NUMBER MmHighMemoryThreshold;
 692 extern PFN_NUMBER MiLowPagedPoolThreshold;
 693 extern PFN_NUMBER MiHighPagedPoolThreshold;
 694 extern PFN_NUMBER MiLowNonPagedPoolThreshold;
 695 extern PFN_NUMBER MiHighNonPagedPoolThreshold;
 696 extern PFN_NUMBER MmMinimumFreePages;
 697 extern PFN_NUMBER MmPlentyFreePages;
 698 extern SIZE_T MmMinimumStackCommitInBytes;
 699 extern PFN_COUNT MiExpansionPoolPagesInitialCharge;
 700 extern PFN_NUMBER MmResidentAvailablePages;
 701 extern PFN_NUMBER MmResidentAvailableAtInit;
 702 extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
 703 extern PFN_NUMBER MmTotalSystemDriverPages;
 704 extern ULONG MmCritsectTimeoutSeconds;
 705 extern PVOID MiSessionImageStart;
 706 extern PVOID MiSessionImageEnd;
 707 extern PMMPTE MiHighestUserPte;
 708 extern PMMPDE MiHighestUserPde;
 709 extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT];
 710 extern PMMPTE MmSharedUserDataPte;
 711 extern LIST_ENTRY MmProcessList;
 712 extern BOOLEAN MmZeroingPageThreadActive;
 713 extern KEVENT MmZeroingPageEvent;
 714 extern ULONG MmSystemPageColor;
 715 extern ULONG MmProcessColorSeed;
 716 extern PMMWSL MmWorkingSetList;
 717 extern PFN_NUMBER MiNumberOfFreePages;
 718 extern SIZE_T MmSessionViewSize;
 719 extern SIZE_T MmSessionPoolSize;
 720 extern SIZE_T MmSessionImageSize;
 721 extern PVOID MiSystemViewStart;
 722 extern PVOID MiSessionPoolEnd;     // 0xBE000000
 723 extern PVOID MiSessionPoolStart;   // 0xBD000000
 724 extern PVOID MiSessionViewStart;   // 0xBE000000
 725 extern PVOID MiSessionSpaceWs;
 726 extern ULONG MmMaximumDeadKernelStacks;
 727 extern SLIST_HEADER MmDeadStackSListHead;
 728 extern MM_AVL_TABLE MmSectionBasedRoot;
 729 extern KGUARDED_MUTEX MmSectionBasedMutex;
 730 extern PVOID MmHighSectionBase;
 731 extern SIZE_T MmSystemLockPagesCount;
 732 extern ULONG_PTR MmSubsectionBase;
 733 extern LARGE_INTEGER MmCriticalSectionTimeout;
 734 extern LIST_ENTRY MmWorkingSetExpansionHead;
 735
 736 FORCEINLINE
 737 BOOLEAN
 738 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType)
 739 {
 740     return ((MemoryType == LoaderFree) ||
 741             (MemoryType == LoaderLoadedProgram) ||
 742             (MemoryType == LoaderFirmwareTemporary) ||
 743             (MemoryType == LoaderOsloaderStack));
 744 }
 745
 746 FORCEINLINE
 747 BOOLEAN
 748 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType)
 749 {
 750     return ((MemoryType == LoaderFirmwarePermanent) ||
 751             (MemoryType == LoaderSpecialMemory) ||
 752             (MemoryType == LoaderHALCachedMemory) ||
 753             (MemoryType == LoaderBBTMemory));
 754 }
 755
 756 #ifdef _M_AMD64
 757 FORCEINLINE
 758 BOOLEAN
 759 MiIsUserPxe(PVOID Address)
 760 {
 761     return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL;
 762 }
 763
 764 FORCEINLINE
 765 BOOLEAN
 766 MiIsUserPpe(PVOID Address)
 767 {
 768     return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL;
 769 }
 770
 771 FORCEINLINE
 772 BOOLEAN
 773 MiIsUserPde(PVOID Address)
 774 {
 775     return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL;
 776 }
 777
 778 FORCEINLINE
 779 BOOLEAN
 780 MiIsUserPte(PVOID Address)
 781 {
 782     return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL;
 783 }
 784 #else
 785 FORCEINLINE
 786 BOOLEAN
 787 MiIsUserPde(PVOID Address)
 788 {
 789     return ((Address >= (PVOID)MiAddressToPde(NULL)) &&
 790             (Address <= (PVOID)MiHighestUserPde));
 791 }
 792
 793 FORCEINLINE
 794 BOOLEAN
 795 MiIsUserPte(PVOID Address)
 796 {
 797     return (Address <= (PVOID)MiHighestUserPte);
 798 }
 799 #endif
 800
 801 //
 802 // Figures out the hardware bits for a PTE
 803 //
 804 FORCEINLINE
 805 ULONG_PTR
 806 MiDetermineUserGlobalPteMask(IN PVOID PointerPte)
 807 {
 808     MMPTE TempPte;
 809
 810     /* Start fresh */
 811     TempPte.u.Long = 0;
 812
 813     /* Make it valid and accessed */
 814     TempPte.u.Hard.Valid = TRUE;
 815     MI_MAKE_ACCESSED_PAGE(&TempPte);
 816
 817     /* Is this for user-mode? */
 818     if (
 819 #if (_MI_PAGING_LEVELS == 4)
 820         MiIsUserPxe(PointerPte) ||
 821 #endif
 822 #if (_MI_PAGING_LEVELS >= 3)
 823         MiIsUserPpe(PointerPte) ||
 824 #endif
 825         MiIsUserPde(PointerPte) ||
 826         MiIsUserPte(PointerPte))
 827     {
 828         /* Set the owner bit */
 829         MI_MAKE_OWNER_PAGE(&TempPte);
 830     }
 831
 832     /* FIXME: We should also set the global bit */
 833
 834     /* Return the protection */
 835     return TempPte.u.Long;
 836 }
 837
 838 //
 839 // Creates a valid kernel PTE with the given protection
 840 //
 841 FORCEINLINE
 842 VOID
 843 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte,
 844                             IN PMMPTE MappingPte,
 845                             IN ULONG_PTR ProtectionMask,
 846                             IN PFN_NUMBER PageFrameNumber)
 847 {
 848     /* Only valid for kernel, non-session PTEs */
 849     ASSERT(MappingPte > MiHighestUserPte);
 850     ASSERT(!MI_IS_SESSION_PTE(MappingPte));
 851     ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP));
 852
 853     /* Start fresh */
 854     *NewPte = ValidKernelPte;
 855
 856     /* Set the protection and page */
 857     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 858     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 859 }
 860
 861 //
 862 // Creates a valid PTE with the given protection
 863 //
 864 FORCEINLINE
 865 VOID
 866 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte,
 867                      IN PMMPTE MappingPte,
 868                      IN ULONG_PTR ProtectionMask,
 869                      IN PFN_NUMBER PageFrameNumber)
 870 {
 871     /* Set the protection and page */
 872     NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte);
 873     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 874     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 875 }
 876
 877 //
 878 // Creates a valid user PTE with the given protection
 879 //
 880 FORCEINLINE
 881 VOID
 882 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte,
 883                           IN PMMPTE MappingPte,
 884                           IN ULONG_PTR ProtectionMask,
 885                           IN PFN_NUMBER PageFrameNumber)
 886 {
 887     /* Only valid for kernel, non-session PTEs */
 888     ASSERT(MappingPte <= MiHighestUserPte);
 889
 890     /* Start fresh */
 891     NewPte->u.Long = 0;
 892
 893     /* Set the protection and page */
 894     NewPte->u.Hard.Valid = TRUE;
 895     NewPte->u.Hard.Owner = TRUE;
 896     NewPte->u.Hard.PageFrameNumber = PageFrameNumber;
 897     NewPte->u.Long |= MmProtectToPteMask[ProtectionMask];
 898 }
 899
 900 #ifndef _M_AMD64
 901 //
 902 // Builds a Prototype PTE for the address of the PTE
 903 //
 904 FORCEINLINE
 905 VOID
 906 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte,
 907                       IN PMMPTE PointerPte)
 908 {
 909     ULONG_PTR Offset;
 910
 911     /* Mark this as a prototype */
 912     NewPte->u.Long = 0;
 913     NewPte->u.Proto.Prototype = 1;
 914
 915     /*
 916      * Prototype PTEs are only valid in paged pool by design, this little trick
 917      * lets us only use 30 bits for the adress of the PTE, as long as the area
 918      * stays 1024MB At most.
 919      */
 920     Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart;
 921
 922     /*
 923      * 7 bits go in the "low" (but we assume the bottom 2 are zero)
 924      * and the other 21 bits go in the "high"
 925      */
 926     NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2;
 927     NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9;
 928 }
 929
 930 //
 931 // Builds a Subsection PTE for the address of the Segment
 932 //
 933 FORCEINLINE
 934 VOID
 935 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte,
 936                        IN PVOID Segment)
 937 {
 938     ULONG_PTR Offset;
 939
 940     /* Mark this as a prototype */
 941     NewPte->u.Long = 0;
 942     NewPte->u.Subsect.Prototype = 1;
 943
 944     /*
 945      * Segments are only valid either in nonpaged pool. We store the 20 bit
 946      * difference either from the top or bottom of nonpaged pool, giving a
 947      * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
 948      * 256MB.
 949      */
 950     if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB)))
 951     {
 952         Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase;
 953         NewPte->u.Subsect.WhichPool = PagedPool;
 954     }
 955     else
 956     {
 957         Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment;
 958         NewPte->u.Subsect.WhichPool = NonPagedPool;
 959     }
 960
 961     /*
 962      * 4 bits go in the "low" (but we assume the bottom 3 are zero)
 963      * and the other 20 bits go in the "high"
 964      */
 965     NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3;
 966     NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7;
 967 }
 968
 969 FORCEINLINE
 970 BOOLEAN
 971 MI_IS_MAPPED_PTE(PMMPTE PointerPte)
 972 {
 973     /// \todo Make this reasonable code, this is UGLY!
 974     return ((PointerPte->u.Long & 0xFFFFFC01) != 0);
 975 }
 976
 977 #endif
 978
 979 //
 980 // Returns if the page is physically resident (ie: a large page)
 981 // FIXFIX: CISC/x86 only?
 982 //
 983 FORCEINLINE
 984 BOOLEAN
 985 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address)
 986 {
 987     PMMPDE PointerPde;
 988
 989     /* Large pages are never paged out, always physically resident */
 990     PointerPde = MiAddressToPde(Address);
 991     return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid));
 992 }
 993
 994 //
 995 // Writes a valid PTE
 996 //
 997 FORCEINLINE
 998 VOID
 999 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte,
1000                    IN MMPTE TempPte)
1001 {
1002     /* Write the valid PTE */
1003     ASSERT(PointerPte->u.Hard.Valid == 0);
1004     ASSERT(TempPte.u.Hard.Valid == 1);
1005     *PointerPte = TempPte;
1006 }
1007
1008 //
1009 // Updates a valid PTE
1010 //
1011 FORCEINLINE
1012 VOID
1013 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte,
1014                    IN MMPTE TempPte)
1015 {
1016     /* Write the valid PTE */
1017     ASSERT(PointerPte->u.Hard.Valid == 1);
1018     ASSERT(TempPte.u.Hard.Valid == 1);
1019     ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber);
1020     *PointerPte = TempPte;
1021 }
1022
1023 //
1024 // Writes an invalid PTE
1025 //
1026 FORCEINLINE
1027 VOID
1028 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte,
1029                      IN MMPTE InvalidPte)
1030 {
1031     /* Write the invalid PTE */
1032     ASSERT(InvalidPte.u.Hard.Valid == 0);
1033     ASSERT(InvalidPte.u.Long != 0);
1034     *PointerPte = InvalidPte;
1035 }
1036
1037 //
1038 // Erase the PTE completely
1039 //
1040 FORCEINLINE
1041 VOID
1042 MI_ERASE_PTE(IN PMMPTE PointerPte)
1043 {
1044     /* Zero out the PTE */
1045     ASSERT(PointerPte->u.Long != 0);
1046     PointerPte->u.Long = 0;
1047 }
1048
1049 //
1050 // Writes a valid PDE
1051 //
1052 FORCEINLINE
1053 VOID
1054 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde,
1055                    IN MMPDE TempPde)
1056 {
1057     /* Write the valid PDE */
1058     ASSERT(PointerPde->u.Hard.Valid == 0);
1059     ASSERT(TempPde.u.Hard.Valid == 1);
1060     *PointerPde = TempPde;
1061 }
1062
1063 //
1064 // Writes an invalid PDE
1065 //
1066 FORCEINLINE
1067 VOID
1068 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde,
1069                      IN MMPDE InvalidPde)
1070 {
1071     /* Write the invalid PDE */
1072     ASSERT(InvalidPde.u.Hard.Valid == 0);
1073     ASSERT(InvalidPde.u.Long != 0);
1074     *PointerPde = InvalidPde;
1075 }
1076
1077 //
1078 // Checks if the thread already owns a working set
1079 //
1080 FORCEINLINE
1081 BOOLEAN
1082 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread)
1083 {
1084     /* If any of these are held, return TRUE */
1085     return ((Thread->OwnsProcessWorkingSetExclusive) ||
1086             (Thread->OwnsProcessWorkingSetShared) ||
1087             (Thread->OwnsSystemWorkingSetExclusive) ||
1088             (Thread->OwnsSystemWorkingSetShared) ||
1089             (Thread->OwnsSessionWorkingSetExclusive) ||
1090             (Thread->OwnsSessionWorkingSetShared));
1091 }
1092
1093 //
1094 // Checks if the process owns the working set lock
1095 //
1096 FORCEINLINE
1097 BOOLEAN
1098 MI_WS_OWNER(IN PEPROCESS Process)
1099 {
1100     /* Check if this process is the owner, and that the thread owns the WS */
1101     if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0)
1102     {
1103         DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1104     }
1105     if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb)
1106     {
1107         DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process);
1108     }
1109     return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) &&
1110             ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) ||
1111              (PsGetCurrentThread()->OwnsProcessWorkingSetShared)));
1112 }
1113
1114 //
1115 // New ARM3<->RosMM PAGE Architecture
1116 //
1117 FORCEINLINE
1118 BOOLEAN
1119 MiIsRosSectionObject(IN PVOID Section)
1120 {
1121     PROS_SECTION_OBJECT RosSection = Section;
1122     if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE;
1123     return FALSE;
1124 }
1125
1126 #define MI_IS_ROS_PFN(x)     ((x)->u4.AweAllocation == TRUE)
1127
1128 VOID
1129 NTAPI
1130 MiDecrementReferenceCount(
1131     IN PMMPFN Pfn1,
1132     IN PFN_NUMBER PageFrameIndex
1133 );
1134
1135 FORCEINLINE
1136 BOOLEAN
1137 MI_IS_WS_UNSAFE(IN PEPROCESS Process)
1138 {
1139     return (Process->Vm.Flags.AcquiredUnsafe == TRUE);
1140 }
1141
1142 //
1143 // Locks the working set for the given process
1144 //
1145 FORCEINLINE
1146 VOID
1147 MiLockProcessWorkingSet(IN PEPROCESS Process,
1148                         IN PETHREAD Thread)
1149 {
1150     /* Shouldn't already be owning the process working set */
1151     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1152     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1153
1154     /* Block APCs, make sure that still nothing is already held */
1155     KeEnterGuardedRegion();
1156     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1157
1158     /* Lock the working set */
1159     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1160
1161     /* Now claim that we own the lock */
1162     ASSERT(!MI_IS_WS_UNSAFE(Process));
1163     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1164     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1165 }
1166
1167 FORCEINLINE
1168 VOID
1169 MiLockProcessWorkingSetShared(IN PEPROCESS Process,
1170                               IN PETHREAD Thread)
1171 {
1172     /* Shouldn't already be owning the process working set */
1173     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1174     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1175
1176     /* Block APCs, make sure that still nothing is already held */
1177     KeEnterGuardedRegion();
1178     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1179
1180     /* Lock the working set */
1181     ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex);
1182
1183     /* Now claim that we own the lock */
1184     ASSERT(!MI_IS_WS_UNSAFE(Process));
1185     ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE);
1186     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1187     Thread->OwnsProcessWorkingSetShared = TRUE;
1188 }
1189
1190 FORCEINLINE
1191 VOID
1192 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1193                               IN PETHREAD Thread)
1194 {
1195     /* Shouldn't already be owning the process working set */
1196     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1197
1198     /* APCs must be blocked, make sure that still nothing is already held */
1199     ASSERT(KeAreAllApcsDisabled() == TRUE);
1200     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1201
1202     /* Lock the working set */
1203     ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex);
1204
1205     /* Now claim that we own the lock */
1206     ASSERT(!MI_IS_WS_UNSAFE(Process));
1207     Process->Vm.Flags.AcquiredUnsafe = 1;
1208     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1209     Thread->OwnsProcessWorkingSetExclusive = TRUE;
1210 }
1211
1212 //
1213 // Unlocks the working set for the given process
1214 //
1215 FORCEINLINE
1216 VOID
1217 MiUnlockProcessWorkingSet(IN PEPROCESS Process,
1218                           IN PETHREAD Thread)
1219 {
1220     /* Make sure we are the owner of a safe acquisition */
1221     ASSERT(MI_WS_OWNER(Process));
1222     ASSERT(!MI_IS_WS_UNSAFE(Process));
1223
1224     /* The thread doesn't own it anymore */
1225     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1226     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1227
1228     /* Release the lock and re-enable APCs */
1229     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1230     KeLeaveGuardedRegion();
1231 }
1232
1233 //
1234 // Unlocks the working set for the given process
1235 //
1236 FORCEINLINE
1237 VOID
1238 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process,
1239                                 IN PETHREAD Thread)
1240 {
1241     /* Make sure we are the owner of a safe acquisition (because shared) */
1242     ASSERT(MI_WS_OWNER(Process));
1243     ASSERT(!MI_IS_WS_UNSAFE(Process));
1244
1245     /* Ensure we are in a shared acquisition */
1246     ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE);
1247     ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE);
1248
1249     /* Don't claim the lock anylonger */
1250     Thread->OwnsProcessWorkingSetShared = FALSE;
1251
1252     /* Release the lock and re-enable APCs */
1253     ExReleasePushLockShared(&Process->Vm.WorkingSetMutex);
1254     KeLeaveGuardedRegion();
1255 }
1256
1257 //
1258 // Unlocks the working set for the given process
1259 //
1260 FORCEINLINE
1261 VOID
1262 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process,
1263                                 IN PETHREAD Thread)
1264 {
1265     /* Make sure we are the owner of an unsafe acquisition */
1266     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1267     ASSERT(KeAreAllApcsDisabled() == TRUE);
1268     ASSERT(MI_WS_OWNER(Process));
1269     ASSERT(MI_IS_WS_UNSAFE(Process));
1270
1271     /* No longer unsafe */
1272     Process->Vm.Flags.AcquiredUnsafe = 0;
1273
1274     /* The thread doesn't own it anymore */
1275     ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE);
1276     Thread->OwnsProcessWorkingSetExclusive = FALSE;
1277
1278     /* Release the lock but don't touch APC state */
1279     ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex);
1280     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1281 }
1282
1283 //
1284 // Locks the working set
1285 //
1286 FORCEINLINE
1287 VOID
1288 MiLockWorkingSet(IN PETHREAD Thread,
1289                  IN PMMSUPPORT WorkingSet)
1290 {
1291     /* Block APCs */
1292     KeEnterGuardedRegion();
1293
1294     /* Working set should be in global memory */
1295     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1296
1297     /* Thread shouldn't already be owning something */
1298     ASSERT(!MM_ANY_WS_LOCK_HELD(Thread));
1299
1300     /* Lock this working set */
1301     ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex);
1302
1303     /* Which working set is this? */
1304     if (WorkingSet == &MmSystemCacheWs)
1305     {
1306         /* Own the system working set */
1307         ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) &&
1308                (Thread->OwnsSystemWorkingSetShared == FALSE));
1309         Thread->OwnsSystemWorkingSetExclusive = TRUE;
1310     }
1311     else if (WorkingSet->Flags.SessionSpace)
1312     {
1313         /* Own the session working set */
1314         ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) &&
1315                (Thread->OwnsSessionWorkingSetShared == FALSE));
1316         Thread->OwnsSessionWorkingSetExclusive = TRUE;
1317     }
1318     else
1319     {
1320         /* Own the process working set */
1321         ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) &&
1322                (Thread->OwnsProcessWorkingSetShared == FALSE));
1323         Thread->OwnsProcessWorkingSetExclusive = TRUE;
1324     }
1325 }
1326
1327 //
1328 // Unlocks the working set
1329 //
1330 FORCEINLINE
1331 VOID
1332 MiUnlockWorkingSet(IN PETHREAD Thread,
1333                    IN PMMSUPPORT WorkingSet)
1334 {
1335     /* Working set should be in global memory */
1336     ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE);
1337
1338     /* Which working set is this? */
1339     if (WorkingSet == &MmSystemCacheWs)
1340     {
1341         /* Release the system working set */
1342         ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) ||
1343                (Thread->OwnsSystemWorkingSetShared == TRUE));
1344         Thread->OwnsSystemWorkingSetExclusive = FALSE;
1345     }
1346     else if (WorkingSet->Flags.SessionSpace)
1347     {
1348         /* Release the session working set */
1349         ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) ||
1350                (Thread->OwnsSessionWorkingSetShared == TRUE));
1351         Thread->OwnsSessionWorkingSetExclusive = 0;
1352     }
1353     else
1354     {
1355         /* Release the process working set */
1356         ASSERT((Thread->OwnsProcessWorkingSetExclusive) ||
1357                (Thread->OwnsProcessWorkingSetShared));
1358         Thread->OwnsProcessWorkingSetExclusive = FALSE;
1359     }
1360
1361     /* Release the working set lock */
1362     ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex);
1363
1364     /* Unblock APCs */
1365     KeLeaveGuardedRegion();
1366 }
1367
1368 FORCEINLINE
1369 VOID
1370 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process,
1371                                   IN PETHREAD Thread,
1372                                   OUT PBOOLEAN Safe,
1373                                   OUT PBOOLEAN Shared)
1374 {
1375     ASSERT(MI_WS_OWNER(Process));
1376
1377     /* Check if the current owner is unsafe */
1378     if (MI_IS_WS_UNSAFE(Process))
1379     {
1380         /* Release unsafely */
1381         MiUnlockProcessWorkingSetUnsafe(Process, Thread);
1382         *Safe = FALSE;
1383         *Shared = FALSE;
1384     }
1385     else if (Thread->OwnsProcessWorkingSetExclusive == 1)
1386     {
1387         /* Owner is safe and exclusive, release normally */
1388         MiUnlockProcessWorkingSet(Process, Thread);
1389         *Safe = TRUE;
1390         *Shared = FALSE;
1391     }
1392     else
1393     {
1394         /* Owner is shared (implies safe), release normally */
1395         MiUnlockProcessWorkingSetShared(Process, Thread);
1396         *Safe = TRUE;
1397         *Shared = TRUE;
1398     }
1399 }
1400
1401 FORCEINLINE
1402 VOID
1403 MiLockProcessWorkingSetForFault(IN PEPROCESS Process,
1404                                 IN PETHREAD Thread,
1405                                 IN BOOLEAN Safe,
1406                                 IN BOOLEAN Shared)
1407 {
1408     /* Check if this was a safe lock or not */
1409     if (Safe)
1410     {
1411         if (Shared)
1412         {
1413             /* Reacquire safely & shared */
1414             MiLockProcessWorkingSetShared(Process, Thread);
1415         }
1416         else
1417         {
1418             /* Reacquire safely */
1419             MiLockProcessWorkingSet(Process, Thread);
1420         }
1421     }
1422     else
1423     {
1424         /* Unsafe lock cannot be shared */
1425         ASSERT(Shared == FALSE);
1426         /* Reacquire unsafely */
1427         MiLockProcessWorkingSetUnsafe(Process, Thread);
1428     }
1429 }
1430
1431 //
1432 // Returns the ProtoPTE inside a VAD for the given VPN
1433 //
1434 FORCEINLINE
1435 PMMPTE
1436 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad,
1437                              IN ULONG_PTR Vpn)
1438 {
1439     PMMPTE ProtoPte;
1440
1441     /* Find the offset within the VAD's prototype PTEs */
1442     ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn);
1443     ASSERT(ProtoPte <= Vad->LastContiguousPte);
1444     return ProtoPte;
1445 }
1446
1447 //
1448 // Returns the PFN Database entry for the given page number
1449 // Warning: This is not necessarily a valid PFN database entry!
1450 //
1451 FORCEINLINE
1452 PMMPFN
1453 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn)
1454 {
1455     /* Get the entry */
1456     return &MmPfnDatabase[Pfn];
1457 };
1458
1459 //
1460 // Drops a locked page without dereferencing it
1461 //
1462 FORCEINLINE
1463 VOID
1464 MiDropLockCount(IN PMMPFN Pfn1)
1465 {
1466     /* This page shouldn't be locked, but it should be valid */
1467     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1468     ASSERT(Pfn1->u2.ShareCount == 0);
1469
1470     /* Is this the last reference to the page */
1471     if (Pfn1->u3.e2.ReferenceCount == 1)
1472     {
1473         /* It better not be valid */
1474         ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1475
1476         /* Is it a prototype PTE? */
1477         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1478             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1479         {
1480             /* FIXME: We should return commit */
1481             DPRINT1("Not returning commit for prototype PTE\n");
1482         }
1483
1484         /* Update the counter */
1485         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1486     }
1487 }
1488
1489 //
1490 // Drops a locked page and dereferences it
1491 //
1492 FORCEINLINE
1493 VOID
1494 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1)
1495 {
1496     USHORT RefCount, OldRefCount;
1497     PFN_NUMBER PageFrameIndex;
1498
1499     /* Loop while we decrement the page successfully */
1500     do
1501     {
1502         /* There should be at least one reference */
1503         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1504         ASSERT(OldRefCount != 0);
1505
1506         /* Are we the last one */
1507         if (OldRefCount == 1)
1508         {
1509             /* The page shoudln't be shared not active at this point */
1510             ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
1511             ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1512             ASSERT(Pfn1->u2.ShareCount == 0);
1513
1514             /* Is it a prototype PTE? */
1515             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1516                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1517             {
1518                 /* FIXME: We should return commit */
1519                 DPRINT1("Not returning commit for prototype PTE\n");
1520             }
1521
1522             /* Update the counter, and drop a reference the long way */
1523             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1524             PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
1525             MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1526             return;
1527         }
1528
1529         /* Drop a reference the short way, and that's it */
1530         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1531                                                 OldRefCount - 1,
1532                                                 OldRefCount);
1533         ASSERT(RefCount != 0);
1534     } while (OldRefCount != RefCount);
1535
1536     /* If we got here, there should be more than one reference */
1537     ASSERT(RefCount > 1);
1538     if (RefCount == 2)
1539     {
1540         /* Is it still being shared? */
1541         if (Pfn1->u2.ShareCount >= 1)
1542         {
1543             /* Then it should be valid */
1544             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1545
1546             /* Is it a prototype PTE? */
1547             if ((Pfn1->u3.e1.PrototypePte == 1) &&
1548                 (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1549             {
1550                 /* We don't handle ethis */
1551                 ASSERT(FALSE);
1552             }
1553
1554             /* Update the counter */
1555             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1556         }
1557     }
1558 }
1559
1560 //
1561 // References a locked page and updates the counter
1562 // Used in MmProbeAndLockPages to handle different edge cases
1563 //
1564 FORCEINLINE
1565 VOID
1566 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1)
1567 {
1568     USHORT RefCount, OldRefCount;
1569
1570     /* Sanity check */
1571     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1572
1573     /* Does ARM3 own the page? */
1574     if (MI_IS_ROS_PFN(Pfn1))
1575     {
1576         /* ReactOS Mm doesn't track share count */
1577         ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1578     }
1579     else
1580     {
1581         /* On ARM3 pages, we should see a valid share count */
1582         ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid));
1583
1584         /* Is it a prototype PTE? */
1585         if ((Pfn1->u3.e1.PrototypePte == 1) &&
1586             (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1587         {
1588             /* FIXME: We should charge commit */
1589             DPRINT1("Not charging commit for prototype PTE\n");
1590         }
1591     }
1592
1593     /* More locked pages! */
1594     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1595
1596     /* Loop trying to update the reference count */
1597     do
1598     {
1599         /* Get the current reference count, make sure it's valid */
1600         OldRefCount = Pfn1->u3.e2.ReferenceCount;
1601         ASSERT(OldRefCount != 0);
1602         ASSERT(OldRefCount < 2500);
1603
1604         /* Bump it up by one */
1605         RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount,
1606                                                 OldRefCount + 1,
1607                                                 OldRefCount);
1608         ASSERT(RefCount != 0);
1609     } while (OldRefCount != RefCount);
1610
1611     /* Was this the first lock attempt? If not, undo our bump */
1612     if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1613 }
1614
1615 //
1616 // References a locked page and updates the counter
1617 // Used in all other cases except MmProbeAndLockPages
1618 //
1619 FORCEINLINE
1620 VOID
1621 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1)
1622 {
1623     USHORT NewRefCount;
1624
1625     /* Is it a prototype PTE? */
1626     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1627         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1628     {
1629         /* FIXME: We should charge commit */
1630         DPRINT1("Not charging commit for prototype PTE\n");
1631     }
1632
1633     /* More locked pages! */
1634     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1635
1636     /* Update the reference count */
1637     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1638     if (NewRefCount == 2)
1639     {
1640         /* Is it locked or shared? */
1641         if (Pfn1->u2.ShareCount)
1642         {
1643             /* It's shared, so make sure it's active */
1644             ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
1645         }
1646         else
1647         {
1648             /* It's locked, so we shouldn't lock again */
1649             InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1650         }
1651     }
1652     else
1653     {
1654         /* Someone had already locked the page, so undo our bump */
1655         ASSERT(NewRefCount < 2500);
1656         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1657     }
1658 }
1659
1660 //
1661 // References a locked page and updates the counter
1662 // Used in all other cases except MmProbeAndLockPages
1663 //
1664 FORCEINLINE
1665 VOID
1666 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1)
1667 {
1668     USHORT NewRefCount;
1669
1670     /* Make sure the page isn't used yet */
1671     ASSERT(Pfn1->u2.ShareCount == 0);
1672     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1673
1674     /* Is it a prototype PTE? */
1675     if ((Pfn1->u3.e1.PrototypePte == 1) &&
1676         (Pfn1->OriginalPte.u.Soft.Prototype == 1))
1677     {
1678         /* FIXME: We should charge commit */
1679         DPRINT1("Not charging commit for prototype PTE\n");
1680     }
1681
1682     /* More locked pages! */
1683     InterlockedIncrementSizeT(&MmSystemLockPagesCount);
1684
1685     /* Update the reference count */
1686     NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1687     if (NewRefCount != 1)
1688     {
1689         /* Someone had already locked the page, so undo our bump */
1690         ASSERT(NewRefCount < 2500);
1691         InterlockedDecrementSizeT(&MmSystemLockPagesCount);
1692     }
1693 }
1694
1695 FORCEINLINE
1696 VOID
1697 MiIncrementPageTableReferences(IN PVOID Address)
1698 {
1699     PUSHORT RefCount;
1700
1701     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1702
1703     *RefCount += 1;
1704     ASSERT(*RefCount <= PTE_PER_PAGE);
1705 }
1706
1707 FORCEINLINE
1708 VOID
1709 MiDecrementPageTableReferences(IN PVOID Address)
1710 {
1711     PUSHORT RefCount;
1712
1713     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1714
1715     *RefCount -= 1;
1716     ASSERT(*RefCount < PTE_PER_PAGE);
1717 }
1718
1719 FORCEINLINE
1720 USHORT
1721 MiQueryPageTableReferences(IN PVOID Address)
1722 {
1723     PUSHORT RefCount;
1724
1725     RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)];
1726
1727     return *RefCount;
1728 }
1729
1730 BOOLEAN
1731 NTAPI
1732 MmArmInitSystem(
1733     IN ULONG Phase,
1734     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1735 );
1736
1737 VOID
1738 NTAPI
1739 MiInitializeSessionSpaceLayout();
1740
1741 NTSTATUS
1742 NTAPI
1743 MiInitMachineDependent(
1744     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1745 );
1746
1747 VOID
1748 NTAPI
1749 MiComputeColorInformation(
1750     VOID
1751 );
1752
1753 VOID
1754 NTAPI
1755 MiMapPfnDatabase(
1756     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1757 );
1758
1759 VOID
1760 NTAPI
1761 MiInitializeColorTables(
1762     VOID
1763 );
1764
1765 VOID
1766 NTAPI
1767 MiInitializePfnDatabase(
1768     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1769 );
1770
1771 VOID
1772 NTAPI
1773 MiInitializeSessionWsSupport(
1774     VOID
1775 );
1776
1777 VOID
1778 NTAPI
1779 MiInitializeSessionIds(
1780     VOID
1781 );
1782
1783 BOOLEAN
1784 NTAPI
1785 MiInitializeMemoryEvents(
1786     VOID
1787 );
1788
1789 PFN_NUMBER
1790 NTAPI
1791 MxGetNextPage(
1792     IN PFN_NUMBER PageCount
1793 );
1794
1795 PPHYSICAL_MEMORY_DESCRIPTOR
1796 NTAPI
1797 MmInitializeMemoryLimits(
1798     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1799     IN PBOOLEAN IncludeType
1800 );
1801
1802 PFN_NUMBER
1803 NTAPI
1804 MiPagesInLoaderBlock(
1805     IN PLOADER_PARAMETER_BLOCK LoaderBlock,
1806     IN PBOOLEAN IncludeType
1807 );
1808
1809 VOID
1810 FASTCALL
1811 MiSyncARM3WithROS(
1812     IN PVOID AddressStart,
1813     IN PVOID AddressEnd
1814 );
1815
1816 NTSTATUS
1817 NTAPI
1818 MiRosProtectVirtualMemory(
1819     IN PEPROCESS Process,
1820     IN OUT PVOID *BaseAddress,
1821     IN OUT PSIZE_T NumberOfBytesToProtect,
1822     IN ULONG NewAccessProtection,
1823     OUT PULONG OldAccessProtection OPTIONAL
1824 );
1825
1826 NTSTATUS
1827 NTAPI
1828 MmArmAccessFault(
1829     IN BOOLEAN StoreInstruction,
1830     IN PVOID Address,
1831     IN KPROCESSOR_MODE Mode,
1832     IN PVOID TrapInformation
1833 );
1834
1835 NTSTATUS
1836 FASTCALL
1837 MiCheckPdeForPagedPool(
1838     IN PVOID Address
1839 );
1840
1841 VOID
1842 NTAPI
1843 MiInitializeNonPagedPool(
1844     VOID
1845 );
1846
1847 VOID
1848 NTAPI
1849 MiInitializeNonPagedPoolThresholds(
1850     VOID
1851 );
1852
1853 VOID
1854 NTAPI
1855 MiInitializePoolEvents(
1856     VOID
1857 );
1858
1859 VOID                      //
1860 NTAPI                     //
1861 InitializePool(           //
1862     IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge
1863     IN ULONG Threshold    //
1864 );                        //
1865
1866 // FIXFIX: THIS ONE TOO
1867 VOID
1868 NTAPI
1869 INIT_FUNCTION
1870 ExInitializePoolDescriptor(
1871     IN PPOOL_DESCRIPTOR PoolDescriptor,
1872     IN POOL_TYPE PoolType,
1873     IN ULONG PoolIndex,
1874     IN ULONG Threshold,
1875     IN PVOID PoolLock
1876 );
1877
1878 NTSTATUS
1879 NTAPI
1880 MiInitializeSessionPool(
1881     VOID
1882 );
1883
1884 VOID
1885 NTAPI
1886 MiInitializeSystemPtes(
1887     IN PMMPTE StartingPte,
1888     IN ULONG NumberOfPtes,
1889     IN MMSYSTEM_PTE_POOL_TYPE PoolType
1890 );
1891
1892 PMMPTE
1893 NTAPI
1894 MiReserveSystemPtes(
1895     IN ULONG NumberOfPtes,
1896     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1897 );
1898
1899 VOID
1900 NTAPI
1901 MiReleaseSystemPtes(
1902     IN PMMPTE StartingPte,
1903     IN ULONG NumberOfPtes,
1904     IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1905 );
1906
1907
1908 PFN_NUMBER
1909 NTAPI
1910 MiFindContiguousPages(
1911     IN PFN_NUMBER LowestPfn,
1912     IN PFN_NUMBER HighestPfn,
1913     IN PFN_NUMBER BoundaryPfn,
1914     IN PFN_NUMBER SizeInPages,
1915     IN MEMORY_CACHING_TYPE CacheType
1916 );
1917
1918 PVOID
1919 NTAPI
1920 MiCheckForContiguousMemory(
1921     IN PVOID BaseAddress,
1922     IN PFN_NUMBER BaseAddressPages,
1923     IN PFN_NUMBER SizeInPages,
1924     IN PFN_NUMBER LowestPfn,
1925     IN PFN_NUMBER HighestPfn,
1926     IN PFN_NUMBER BoundaryPfn,
1927     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1928 );
1929
1930 PMDL
1931 NTAPI
1932 MiAllocatePagesForMdl(
1933     IN PHYSICAL_ADDRESS LowAddress,
1934     IN PHYSICAL_ADDRESS HighAddress,
1935     IN PHYSICAL_ADDRESS SkipBytes,
1936     IN SIZE_T TotalBytes,
1937     IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
1938     IN ULONG Flags
1939 );
1940
1941 PVOID
1942 NTAPI
1943 MiMapLockedPagesInUserSpace(
1944     IN PMDL Mdl,
1945     IN PVOID BaseVa,
1946     IN MEMORY_CACHING_TYPE CacheType,
1947     IN PVOID BaseAddress
1948 );
1949
1950 VOID
1951 NTAPI
1952 MiUnmapLockedPagesInUserSpace(
1953     IN PVOID BaseAddress,
1954     IN PMDL Mdl
1955 );
1956
1957 VOID
1958 NTAPI
1959 MiInsertPageInList(
1960     IN PMMPFNLIST ListHead,
1961     IN PFN_NUMBER PageFrameIndex
1962 );
1963
1964 VOID
1965 NTAPI
1966 MiUnlinkFreeOrZeroedPage(
1967     IN PMMPFN Entry
1968 );
1969
1970 VOID
1971 NTAPI
1972 MiUnlinkPageFromList(
1973     IN PMMPFN Pfn
1974 );
1975
1976 PFN_NUMBER
1977 NTAPI
1978 MiAllocatePfn(
1979     IN PMMPTE PointerPte,
1980     IN ULONG Protection
1981 );
1982
1983 VOID
1984 NTAPI
1985 MiInitializePfn(
1986     IN PFN_NUMBER PageFrameIndex,
1987     IN PMMPTE PointerPte,
1988     IN BOOLEAN Modified
1989 );
1990
1991 NTSTATUS
1992 NTAPI
1993 MiInitializeAndChargePfn(
1994     OUT PPFN_NUMBER PageFrameIndex,
1995     IN PMMPTE PointerPde,
1996     IN PFN_NUMBER ContainingPageFrame,
1997     IN BOOLEAN SessionAllocation
1998 );
1999
2000 VOID
2001 NTAPI
2002 MiInitializePfnAndMakePteValid(
2003     IN PFN_NUMBER PageFrameIndex,
2004     IN PMMPTE PointerPte,
2005     IN MMPTE TempPte
2006 );
2007
2008 VOID
2009 NTAPI
2010 MiInitializePfnForOtherProcess(
2011     IN PFN_NUMBER PageFrameIndex,
2012     IN PMMPTE PointerPte,
2013     IN PFN_NUMBER PteFrame
2014 );
2015
2016 VOID
2017 NTAPI
2018 MiDecrementShareCount(
2019     IN PMMPFN Pfn1,
2020     IN PFN_NUMBER PageFrameIndex
2021 );
2022
2023 PFN_NUMBER
2024 NTAPI
2025 MiRemoveAnyPage(
2026     IN ULONG Color
2027 );
2028
2029 PFN_NUMBER
2030 NTAPI
2031 MiRemoveZeroPage(
2032     IN ULONG Color
2033 );
2034
2035 VOID
2036 NTAPI
2037 MiZeroPhysicalPage(
2038     IN PFN_NUMBER PageFrameIndex
2039 );
2040
2041 VOID
2042 NTAPI
2043 MiInsertPageInFreeList(
2044     IN PFN_NUMBER PageFrameIndex
2045 );
2046
2047 PFN_COUNT
2048 NTAPI
2049 MiDeleteSystemPageableVm(
2050     IN PMMPTE PointerPte,
2051     IN PFN_NUMBER PageCount,
2052     IN ULONG Flags,
2053     OUT PPFN_NUMBER ValidPages
2054 );
2055
2056 ULONG
2057 NTAPI
2058 MiGetPageProtection(
2059     IN PMMPTE PointerPte
2060 );
2061
2062 PLDR_DATA_TABLE_ENTRY
2063 NTAPI
2064 MiLookupDataTableEntry(
2065     IN PVOID Address
2066 );
2067
2068 VOID
2069 NTAPI
2070 MiInitializeDriverLargePageList(
2071     VOID
2072 );
2073
2074 VOID
2075 NTAPI
2076 MiInitializeLargePageSupport(
2077     VOID
2078 );
2079
2080 VOID
2081 NTAPI
2082 MiSyncCachedRanges(
2083     VOID
2084 );
2085
2086 BOOLEAN
2087 NTAPI
2088 MiIsPfnInUse(
2089     IN PMMPFN Pfn1
2090 );
2091
2092 PMMVAD
2093 NTAPI
2094 MiLocateAddress(
2095     IN PVOID VirtualAddress
2096 );
2097
2098 TABLE_SEARCH_RESULT
2099 NTAPI
2100 MiCheckForConflictingNode(
2101     IN ULONG_PTR StartVpn,
2102     IN ULONG_PTR EndVpn,
2103     IN PMM_AVL_TABLE Table,
2104     OUT PMMADDRESS_NODE *NodeOrParent
2105 );
2106
2107 TABLE_SEARCH_RESULT
2108 NTAPI
2109 MiFindEmptyAddressRangeDownTree(
2110     IN SIZE_T Length,
2111     IN ULONG_PTR BoundaryAddress,
2112     IN ULONG_PTR Alignment,
2113     IN PMM_AVL_TABLE Table,
2114     OUT PULONG_PTR Base,
2115     OUT PMMADDRESS_NODE *Parent
2116 );
2117
2118 NTSTATUS
2119 NTAPI
2120 MiFindEmptyAddressRangeDownBasedTree(
2121     IN SIZE_T Length,
2122     IN ULONG_PTR BoundaryAddress,
2123     IN ULONG_PTR Alignment,
2124     IN PMM_AVL_TABLE Table,
2125     OUT PULONG_PTR Base
2126 );
2127
2128 TABLE_SEARCH_RESULT
2129 NTAPI
2130 MiFindEmptyAddressRangeInTree(
2131     IN SIZE_T Length,
2132     IN ULONG_PTR Alignment,
2133     IN PMM_AVL_TABLE Table,
2134     OUT PMMADDRESS_NODE *PreviousVad,
2135     OUT PULONG_PTR Base
2136 );
2137
2138 NTSTATUS
2139 NTAPI
2140 MiCheckSecuredVad(
2141     IN PMMVAD Vad,
2142     IN PVOID Base,
2143     IN SIZE_T Size,
2144     IN ULONG ProtectionMask
2145 );
2146
2147 VOID
2148 NTAPI
2149 MiInsertVad(
2150     IN PMMVAD Vad,
2151     IN PEPROCESS Process
2152 );
2153
2154 VOID
2155 NTAPI
2156 MiInsertBasedSection(
2157     IN PSECTION Section
2158 );
2159
2160 NTSTATUS
2161 NTAPI
2162 MiUnmapViewOfSection(
2163     IN PEPROCESS Process,
2164     IN PVOID BaseAddress,
2165     IN ULONG Flags
2166 );
2167
2168 NTSTATUS
2169 NTAPI
2170 MiRosUnmapViewOfSection(
2171     IN PEPROCESS Process,
2172     IN PVOID BaseAddress,
2173     IN ULONG Flags
2174 );
2175
2176 VOID
2177 NTAPI
2178 MiInsertNode(
2179     IN PMM_AVL_TABLE Table,
2180     IN PMMADDRESS_NODE NewNode,
2181     PMMADDRESS_NODE Parent,
2182     TABLE_SEARCH_RESULT Result
2183 );
2184
2185 VOID
2186 NTAPI
2187 MiRemoveNode(
2188     IN PMMADDRESS_NODE Node,
2189     IN PMM_AVL_TABLE Table
2190 );
2191
2192 PMMADDRESS_NODE
2193 NTAPI
2194 MiGetPreviousNode(
2195     IN PMMADDRESS_NODE Node
2196 );
2197
2198 PMMADDRESS_NODE
2199 NTAPI
2200 MiGetNextNode(
2201     IN PMMADDRESS_NODE Node
2202 );
2203
2204 BOOLEAN
2205 NTAPI
2206 MiInitializeSystemSpaceMap(
2207     IN PMMSESSION InputSession OPTIONAL
2208 );
2209
2210 VOID
2211 NTAPI
2212 MiSessionRemoveProcess(
2213     VOID
2214 );
2215
2216 VOID
2217 NTAPI
2218 MiReleaseProcessReferenceToSessionDataPage(
2219     IN PMM_SESSION_SPACE SessionGlobal
2220 );
2221
2222 VOID
2223 NTAPI
2224 MiSessionAddProcess(
2225     IN PEPROCESS NewProcess
2226 );
2227
2228 NTSTATUS
2229 NTAPI
2230 MiSessionCommitPageTables(
2231     IN PVOID StartVa,
2232     IN PVOID EndVa
2233 );
2234
2235 ULONG
2236 NTAPI
2237 MiMakeProtectionMask(
2238     IN ULONG Protect
2239 );
2240
2241 VOID
2242 NTAPI
2243 MiDeleteVirtualAddresses(
2244     IN ULONG_PTR Va,
2245     IN ULONG_PTR EndingAddress,
2246     IN PMMVAD Vad
2247 );
2248
2249 ULONG
2250 NTAPI
2251 MiMakeSystemAddressValid(
2252     IN PVOID PageTableVirtualAddress,
2253     IN PEPROCESS CurrentProcess
2254 );
2255
2256 ULONG
2257 NTAPI
2258 MiMakeSystemAddressValidPfn(
2259     IN PVOID VirtualAddress,
2260     IN KIRQL OldIrql
2261 );
2262
2263 VOID
2264 NTAPI
2265 MiRemoveMappedView(
2266     IN PEPROCESS CurrentProcess,
2267     IN PMMVAD Vad
2268 );
2269
2270 PSUBSECTION
2271 NTAPI
2272 MiLocateSubsection(
2273     IN PMMVAD Vad,
2274     IN ULONG_PTR Vpn
2275 );
2276
2277 VOID
2278 NTAPI
2279 MiDeleteARM3Section(
2280     PVOID ObjectBody
2281 );
2282
2283 NTSTATUS
2284 NTAPI
2285 MiQueryMemorySectionName(
2286     IN HANDLE ProcessHandle,
2287     IN PVOID BaseAddress,
2288     OUT PVOID MemoryInformation,
2289     IN SIZE_T MemoryInformationLength,
2290     OUT PSIZE_T ReturnLength
2291 );
2292
2293 NTSTATUS
2294 NTAPI
2295 MiRosUnmapViewInSystemSpace(
2296     IN PVOID MappedBase
2297 );
2298
2299 POOL_TYPE
2300 NTAPI
2301 MmDeterminePoolType(
2302     IN PVOID PoolAddress
2303 );
2304
2305 VOID
2306 NTAPI
2307 MiMakePdeExistAndMakeValid(
2308     IN PMMPTE PointerPde,
2309     IN PEPROCESS TargetProcess,
2310     IN KIRQL OldIrql
2311 );
2312
2313 //
2314 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2315 // free pages are available. In some scenarios, we don't/can't run that piece of
2316 // code and would rather only have a real zero page. If we can't have a zero page,
2317 // then we'd like to have our own code to grab a free page and zero it out, by
2318 // using MiRemoveAnyPage. This macro implements this.
2319 //
2320 FORCEINLINE
2321 PFN_NUMBER
2322 MiRemoveZeroPageSafe(IN ULONG Color)
2323 {
2324     if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
2325     return 0;
2326 }
2327
2328 /* EOF */