ntoskrnl/include/internal/mm.h

   1 #pragma once
   2
   3 #include <internal/arch/mm.h>
   4
   5 #ifdef __cplusplus
   6 extern "C" {
   7 #endif
   8
   9 /* TYPES *********************************************************************/
  10
  11 struct _EPROCESS;
  12
  13 extern PMMSUPPORT MmKernelAddressSpace;
  14 extern PFN_COUNT MiFreeSwapPages;
  15 extern PFN_COUNT MiUsedSwapPages;
  16 extern PFN_COUNT MmNumberOfPhysicalPages;
  17 extern UCHAR MmDisablePagingExecutive;
  18 extern PFN_NUMBER MmLowestPhysicalPage;
  19 extern PFN_NUMBER MmHighestPhysicalPage;
  20 extern PFN_NUMBER MmAvailablePages;
  21 extern PFN_NUMBER MmResidentAvailablePages;
  22 extern ULONG MmThrottleTop;
  23 extern ULONG MmThrottleBottom;
  24
  25 extern LIST_ENTRY MmLoadedUserImageList;
  26
  27 extern KMUTANT MmSystemLoadLock;
  28
  29 extern ULONG MmNumberOfPagingFiles;
  30
  31 extern SIZE_T MmTotalNonPagedPoolQuota;
  32 extern SIZE_T MmTotalPagedPoolQuota;
  33
  34 extern PVOID MmUnloadedDrivers;
  35 extern PVOID MmLastUnloadedDrivers;
  36 extern PVOID MmTriageActionTaken;
  37 extern PVOID KernelVerifier;
  38 extern MM_DRIVER_VERIFIER_DATA MmVerifierData;
  39
  40 extern SIZE_T MmTotalCommitLimit;
  41 extern SIZE_T MmTotalCommittedPages;
  42 extern SIZE_T MmSharedCommit;
  43 extern SIZE_T MmDriverCommit;
  44 extern SIZE_T MmProcessCommit;
  45 extern SIZE_T MmPagedPoolCommit;
  46 extern SIZE_T MmPeakCommitment;
  47 extern SIZE_T MmtotalCommitLimitMaximum;
  48
  49 extern PVOID MiDebugMapping; // internal
  50 extern PMMPTE MmDebugPte; // internal
  51
  52 extern KSPIN_LOCK MmPfnLock;
  53
  54 struct _KTRAP_FRAME;
  55 struct _EPROCESS;
  56 struct _MM_RMAP_ENTRY;
  57 typedef ULONG_PTR SWAPENTRY;
  58
  59 //
  60 // Pool Quota values
  61 //
  62 #define MI_QUOTA_NON_PAGED_NEEDED_PAGES             64
  63 #define MI_NON_PAGED_QUOTA_MIN_RESIDENT_PAGES       200
  64 #define MI_CHARGE_PAGED_POOL_QUOTA                  0x80000
  65 #define MI_CHARGE_NON_PAGED_POOL_QUOTA              0x10000
  66
  67 //
  68 // Special IRQL value (found in assertions)
  69 //
  70 #define MM_NOIRQL ((KIRQL)0xFFFFFFFF)
  71
  72 //
  73 // MmDbgCopyMemory Flags
  74 //
  75 #define MMDBG_COPY_WRITE            0x00000001
  76 #define MMDBG_COPY_PHYSICAL         0x00000002
  77 #define MMDBG_COPY_UNSAFE           0x00000004
  78 #define MMDBG_COPY_CACHED           0x00000008
  79 #define MMDBG_COPY_UNCACHED         0x00000010
  80 #define MMDBG_COPY_WRITE_COMBINED   0x00000020
  81
  82 //
  83 // Maximum chunk size per copy
  84 //
  85 #define MMDBG_COPY_MAX_SIZE         0x8
  86
  87 #if defined(_X86_) // intenal for marea.c
  88 #define MI_STATIC_MEMORY_AREAS              (14)
  89 #else
  90 #define MI_STATIC_MEMORY_AREAS              (13)
  91 #endif
  92
  93 #define MEMORY_AREA_SECTION_VIEW            (1)
  94 #ifdef NEWCC
  95 #define MEMORY_AREA_CACHE                   (2)
  96 #endif
  97 #define MEMORY_AREA_OWNED_BY_ARM3           (15)
  98 #define MEMORY_AREA_STATIC                  (0x80000000)
  99
 100 /* Although Microsoft says this isn't hardcoded anymore,
 101    they won't be able to change it. Stuff depends on it */
 102 #define MM_VIRTMEM_GRANULARITY              (64 * 1024)
 103
 104 #define STATUS_MM_RESTART_OPERATION         ((NTSTATUS)0xD0000001)
 105
 106 /*
 107  * Additional flags for protection attributes
 108  */
 109 #define PAGE_WRITETHROUGH                   (1024)
 110 #define PAGE_SYSTEM                         (2048)
 111
 112 #define SEC_PHYSICALMEMORY                  (0x80000000)
 113
 114 #define MC_USER                             (0)
 115 #define MC_SYSTEM                           (1)
 116 #define MC_MAXIMUM                          (2)
 117
 118 #define PAGED_POOL_MASK                     1
 119 #define MUST_SUCCEED_POOL_MASK              2
 120 #define CACHE_ALIGNED_POOL_MASK             4
 121 #define QUOTA_POOL_MASK                     8
 122 #define SESSION_POOL_MASK                   32
 123 #define VERIFIER_POOL_MASK                  64
 124
 125 #define MAX_PAGING_FILES                    (16)
 126
 127 // FIXME: use ALIGN_UP_BY
 128 #define MM_ROUND_UP(x,s)                    \
 129     ((PVOID)(((ULONG_PTR)(x)+(s)-1) & ~((ULONG_PTR)(s)-1)))
 130
 131 #define MM_ROUND_DOWN(x,s)                  \
 132     ((PVOID)(((ULONG_PTR)(x)) & ~((ULONG_PTR)(s)-1)))
 133
 134 #define PAGE_FLAGS_VALID_FOR_SECTION \
 135     (PAGE_READONLY | \
 136      PAGE_READWRITE | \
 137      PAGE_WRITECOPY | \
 138      PAGE_EXECUTE | \
 139      PAGE_EXECUTE_READ | \
 140      PAGE_EXECUTE_READWRITE | \
 141      PAGE_EXECUTE_WRITECOPY | \
 142      PAGE_NOACCESS | \
 143      PAGE_NOCACHE)
 144
 145 #define PAGE_IS_READABLE                    \
 146     (PAGE_READONLY | \
 147     PAGE_READWRITE | \
 148     PAGE_WRITECOPY | \
 149     PAGE_EXECUTE_READ | \
 150     PAGE_EXECUTE_READWRITE | \
 151     PAGE_EXECUTE_WRITECOPY)
 152
 153 #define PAGE_IS_WRITABLE                    \
 154     (PAGE_READWRITE | \
 155     PAGE_WRITECOPY | \
 156     PAGE_EXECUTE_READWRITE | \
 157     PAGE_EXECUTE_WRITECOPY)
 158
 159 #define PAGE_IS_EXECUTABLE                  \
 160     (PAGE_EXECUTE | \
 161     PAGE_EXECUTE_READ | \
 162     PAGE_EXECUTE_READWRITE | \
 163     PAGE_EXECUTE_WRITECOPY)
 164
 165 #define PAGE_IS_WRITECOPY                   \
 166     (PAGE_WRITECOPY | \
 167     PAGE_EXECUTE_WRITECOPY)
 168
 169 //
 170 // Wait entry for marking pages that are being serviced
 171 //
 172 #ifdef _M_IX86
 173 #define MM_WAIT_ENTRY            0x7ffffc00
 174 #elif defined(_M_AMD64)
 175 #define MM_WAIT_ENTRY            0x7FFFFFFFFFFFFC00ULL
 176 #else
 177 #error Unsupported architecture!
 178 #endif
 179
 180 #ifdef _M_AMD64
 181 #define InterlockedCompareExchangePte(PointerPte, Exchange, Comperand) \
 182     InterlockedCompareExchange64((PLONG64)(PointerPte), Exchange, Comperand)
 183
 184 #define InterlockedExchangePte(PointerPte, Value) \
 185     InterlockedExchange64((PLONG64)(PointerPte), Value)
 186 #else
 187 #define InterlockedCompareExchangePte(PointerPte, Exchange, Comperand) \
 188     InterlockedCompareExchange((PLONG)(PointerPte), Exchange, Comperand)
 189
 190 #define InterlockedExchangePte(PointerPte, Value) \
 191     InterlockedExchange((PLONG)(PointerPte), Value)
 192 #endif
 193
 194 typedef struct _MM_SECTION_SEGMENT
 195 {
 196     LONG64 RefCount;
 197     PFILE_OBJECT FileObject;
 198
 199     FAST_MUTEX Lock;            /* lock which protects the page directory */
 200     LARGE_INTEGER RawLength;            /* length of the segment which is part of the mapped file */
 201     LARGE_INTEGER Length;                       /* absolute length of the segment */
 202     PLONG64 ReferenceCount;
 203         ULONG SectionCount;
 204     ULONG Protection;
 205     PULONG Flags;
 206     BOOLEAN WriteCopy;
 207         BOOLEAN Locked;
 208
 209         struct
 210         {
 211                 ULONGLONG FileOffset;           /* start offset into the file for image sections */
 212                 ULONG_PTR VirtualAddress;       /* start offset into the address range for image sections */
 213                 ULONG Characteristics;
 214         } Image;
 215
 216         ULONG SegFlags;
 217
 218     ULONGLONG LastPage;
 219
 220         RTL_GENERIC_TABLE PageTable;
 221 } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT;
 222
 223 typedef struct _MM_IMAGE_SECTION_OBJECT
 224 {
 225     LONG64 RefCount;
 226     PFILE_OBJECT FileObject;
 227     ULONG SectionCount;
 228     LONG MapCount;
 229     ULONG SegFlags;
 230
 231     SECTION_IMAGE_INFORMATION ImageInformation;
 232     PVOID BasedAddress;
 233     ULONG NrSegments;
 234     PMM_SECTION_SEGMENT Segments;
 235 } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT;
 236
 237 #define MM_PHYSICALMEMORY_SEGMENT           (0x1)
 238 #define MM_DATAFILE_SEGMENT                 (0x2)
 239 #define MM_SEGMENT_INDELETE                 (0x4)
 240 #define MM_SEGMENT_INCREATE                 (0x8)
 241 #define MM_IMAGE_SECTION_FLUSH_DELETE       (0x10)
 242
 243
 244 #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT)
 245 #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT)
 246
 247 typedef struct _MEMORY_AREA
 248 {
 249     MMVAD VadNode;
 250
 251     ULONG Type;
 252     ULONG Flags;
 253     BOOLEAN DeleteInProgress;
 254     ULONG Magic;
 255     PVOID Vad;
 256
 257     struct
 258     {
 259         LONGLONG ViewOffset;
 260         PMM_SECTION_SEGMENT Segment;
 261         LIST_ENTRY RegionListHead;
 262     } SectionData;
 263 } MEMORY_AREA, *PMEMORY_AREA;
 264
 265 typedef struct _MM_RMAP_ENTRY
 266 {
 267    struct _MM_RMAP_ENTRY* Next;
 268    PEPROCESS Process;
 269    PVOID Address;
 270 #if DBG
 271    PVOID Caller;
 272 #endif
 273 }
 274 MM_RMAP_ENTRY, *PMM_RMAP_ENTRY;
 275
 276 #if MI_TRACE_PFNS
 277 extern ULONG MI_PFN_CURRENT_USAGE;
 278 extern CHAR MI_PFN_CURRENT_PROCESS_NAME[16];
 279 #define MI_SET_USAGE(x)     MI_PFN_CURRENT_USAGE = x
 280 #define MI_SET_PROCESS2(x)  memcpy(MI_PFN_CURRENT_PROCESS_NAME, x, min(sizeof(x), sizeof(MI_PFN_CURRENT_PROCESS_NAME)))
 281 FORCEINLINE
 282 void
 283 MI_SET_PROCESS(PEPROCESS Process)
 284 {
 285     if (!Process)
 286         MI_SET_PROCESS2("Kernel");
 287     else if (Process == (PEPROCESS)1)
 288         MI_SET_PROCESS2("Hydra");
 289     else
 290         MI_SET_PROCESS2(Process->ImageFileName);
 291 }
 292
 293 FORCEINLINE
 294 void
 295 MI_SET_PROCESS_USTR(PUNICODE_STRING ustr)
 296 {
 297     PWSTR pos, strEnd;
 298     ULONG i;
 299
 300     if (!ustr->Buffer || ustr->Length == 0)
 301     {
 302         MI_PFN_CURRENT_PROCESS_NAME[0] = 0;
 303         return;
 304     }
 305
 306     pos = strEnd = &ustr->Buffer[ustr->Length / sizeof(WCHAR)];
 307     while ((*pos != L'\\') && (pos >  ustr->Buffer))
 308         pos--;
 309
 310     if (*pos == L'\\')
 311         pos++;
 312
 313     for (i = 0; i < sizeof(MI_PFN_CURRENT_PROCESS_NAME) && pos <= strEnd; i++, pos++)
 314         MI_PFN_CURRENT_PROCESS_NAME[i] = (CHAR)*pos;
 315 }
 316 #else
 317 #define MI_SET_USAGE(x)
 318 #define MI_SET_PROCESS(x)
 319 #define MI_SET_PROCESS2(x)
 320 #endif
 321
 322 typedef enum _MI_PFN_USAGES
 323 {
 324     MI_USAGE_NOT_SET = 0,
 325     MI_USAGE_PAGED_POOL,
 326     MI_USAGE_NONPAGED_POOL,
 327     MI_USAGE_NONPAGED_POOL_EXPANSION,
 328     MI_USAGE_KERNEL_STACK,
 329     MI_USAGE_KERNEL_STACK_EXPANSION,
 330     MI_USAGE_SYSTEM_PTE,
 331     MI_USAGE_VAD,
 332     MI_USAGE_PEB_TEB,
 333     MI_USAGE_SECTION,
 334     MI_USAGE_PAGE_TABLE,
 335     MI_USAGE_PAGE_DIRECTORY,
 336     MI_USAGE_LEGACY_PAGE_DIRECTORY,
 337     MI_USAGE_DRIVER_PAGE,
 338     MI_USAGE_CONTINOUS_ALLOCATION,
 339     MI_USAGE_MDL,
 340     MI_USAGE_DEMAND_ZERO,
 341     MI_USAGE_ZERO_LOOP,
 342     MI_USAGE_CACHE,
 343     MI_USAGE_PFN_DATABASE,
 344     MI_USAGE_BOOT_DRIVER,
 345     MI_USAGE_INIT_MEMORY,
 346     MI_USAGE_PAGE_FILE,
 347     MI_USAGE_COW,
 348     MI_USAGE_WSLE,
 349     MI_USAGE_FREE_PAGE
 350 } MI_PFN_USAGES;
 351
 352 //
 353 // These two mappings are actually used by Windows itself, based on the ASSERTS
 354 //
 355 #define StartOfAllocation ReadInProgress
 356 #define EndOfAllocation WriteInProgress
 357
 358 typedef struct _MMPFNENTRY
 359 {
 360     USHORT Modified:1;
 361     USHORT ReadInProgress:1;                 // StartOfAllocation
 362     USHORT WriteInProgress:1;                // EndOfAllocation
 363     USHORT PrototypePte:1;
 364     USHORT PageColor:4;
 365     USHORT PageLocation:3;
 366     USHORT RemovalRequested:1;
 367     USHORT CacheAttribute:2;
 368     USHORT Rom:1;
 369     USHORT ParityError:1;
 370 } MMPFNENTRY;
 371
 372 // Mm internal
 373 typedef struct _MMPFN
 374 {
 375     union
 376     {
 377         PFN_NUMBER Flink;
 378         ULONG WsIndex;
 379         PKEVENT Event;
 380         NTSTATUS ReadStatus;
 381         SINGLE_LIST_ENTRY NextStackPfn;
 382
 383         // HACK for ROSPFN
 384         SWAPENTRY SwapEntry;
 385     } u1;
 386     PMMPTE PteAddress;
 387     union
 388     {
 389         PFN_NUMBER Blink;
 390         ULONG_PTR ShareCount;
 391     } u2;
 392     union
 393     {
 394         struct
 395         {
 396             USHORT ReferenceCount;
 397             MMPFNENTRY e1;
 398         };
 399         struct
 400         {
 401             USHORT ReferenceCount;
 402             USHORT ShortFlags;
 403         } e2;
 404     } u3;
 405     union
 406     {
 407         MMPTE OriginalPte;
 408         LONG AweReferenceCount;
 409
 410         // HACK for ROSPFN
 411         PMM_RMAP_ENTRY RmapListHead;
 412     };
 413     union
 414     {
 415         ULONG_PTR EntireFrame;
 416         struct
 417         {
 418             ULONG_PTR PteFrame:25;
 419             ULONG_PTR InPageError:1;
 420             ULONG_PTR VerifierAllocation:1;
 421             ULONG_PTR AweAllocation:1;
 422             ULONG_PTR Priority:3;
 423             ULONG_PTR MustBeCached:1;
 424         };
 425     } u4;
 426 #if MI_TRACE_PFNS
 427     MI_PFN_USAGES PfnUsage;
 428     CHAR ProcessName[16];
 429 #define MI_SET_PFN_PROCESS_NAME(pfn, x) memcpy(pfn->ProcessName, x, min(sizeof(x), sizeof(pfn->ProcessName)))
 430     PVOID CallSite;
 431 #endif
 432
 433     // HACK until WS lists are supported
 434     MMWSLE Wsle;
 435     struct _MMPFN* NextLRU;
 436     struct _MMPFN* PreviousLRU;
 437 } MMPFN, *PMMPFN;
 438
 439 extern PMMPFN MmPfnDatabase;
 440
 441 typedef struct _MMPFNLIST
 442 {
 443     PFN_NUMBER Total;
 444     MMLISTS ListName;
 445     PFN_NUMBER Flink;
 446     PFN_NUMBER Blink;
 447 } MMPFNLIST, *PMMPFNLIST;
 448
 449 extern MMPFNLIST MmZeroedPageListHead;
 450 extern MMPFNLIST MmFreePageListHead;
 451 extern MMPFNLIST MmStandbyPageListHead;
 452 extern MMPFNLIST MmModifiedPageListHead;
 453 extern MMPFNLIST MmModifiedNoWritePageListHead;
 454
 455 typedef struct _MM_MEMORY_CONSUMER
 456 {
 457     ULONG PagesUsed;
 458     ULONG PagesTarget;
 459     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed);
 460 } MM_MEMORY_CONSUMER, *PMM_MEMORY_CONSUMER;
 461
 462 typedef struct _MM_REGION
 463 {
 464     ULONG Type;
 465     ULONG Protect;
 466     SIZE_T Length;
 467     LIST_ENTRY RegionListEntry;
 468 } MM_REGION, *PMM_REGION;
 469
 470 // Mm internal
 471 /* Entry describing free pool memory */
 472 typedef struct _MMFREE_POOL_ENTRY
 473 {
 474     LIST_ENTRY List;
 475     PFN_COUNT Size;
 476     ULONG Signature;
 477     struct _MMFREE_POOL_ENTRY *Owner;
 478 } MMFREE_POOL_ENTRY, *PMMFREE_POOL_ENTRY;
 479
 480 /* Signature of a freed block */
 481 #define MM_FREE_POOL_SIGNATURE 'ARM3'
 482
 483 /* Paged pool information */
 484 typedef struct _MM_PAGED_POOL_INFO
 485 {
 486     PRTL_BITMAP PagedPoolAllocationMap;
 487     PRTL_BITMAP EndOfPagedPoolBitmap;
 488     PMMPTE FirstPteForPagedPool;
 489     PMMPTE LastPteForPagedPool;
 490     PMMPDE NextPdeForPagedPoolExpansion;
 491     ULONG PagedPoolHint;
 492     SIZE_T PagedPoolCommit;
 493     SIZE_T AllocatedPagedPool;
 494 } MM_PAGED_POOL_INFO, *PMM_PAGED_POOL_INFO;
 495
 496 extern MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
 497
 498 /* Page file information */
 499 typedef struct _MMPAGING_FILE
 500 {
 501     PFN_NUMBER Size;
 502     PFN_NUMBER MaximumSize;
 503     PFN_NUMBER MinimumSize;
 504     PFN_NUMBER FreeSpace;
 505     PFN_NUMBER CurrentUsage;
 506     PFILE_OBJECT FileObject;
 507     UNICODE_STRING PageFileName;
 508     PRTL_BITMAP Bitmap;
 509     HANDLE FileHandle;
 510 }
 511 MMPAGING_FILE, *PMMPAGING_FILE;
 512
 513 extern PMMPAGING_FILE MmPagingFile[MAX_PAGING_FILES];
 514
 515 typedef VOID
 516 (*PMM_ALTER_REGION_FUNC)(
 517     PMMSUPPORT AddressSpace,
 518     PVOID BaseAddress,
 519     SIZE_T Length,
 520     ULONG OldType,
 521     ULONG OldProtect,
 522     ULONG NewType,
 523     ULONG NewProtect
 524 );
 525
 526 typedef VOID
 527 (*PMM_FREE_PAGE_FUNC)(
 528     PVOID Context,
 529     PMEMORY_AREA MemoryArea,
 530     PVOID Address,
 531     PFN_NUMBER Page,
 532     SWAPENTRY SwapEntry,
 533     BOOLEAN Dirty
 534 );
 535
 536 //
 537 // Mm copy support for Kd
 538 //
 539 NTSTATUS
 540 NTAPI
 541 MmDbgCopyMemory(
 542     IN ULONG64 Address,
 543     IN PVOID Buffer,
 544     IN ULONG Size,
 545     IN ULONG Flags
 546 );
 547
 548 //
 549 // Determines if a given address is a session address
 550 //
 551 BOOLEAN
 552 NTAPI
 553 MmIsSessionAddress(
 554     IN PVOID Address
 555 );
 556
 557 ULONG
 558 NTAPI
 559 MmGetSessionId(
 560     IN PEPROCESS Process
 561 );
 562
 563 ULONG
 564 NTAPI
 565 MmGetSessionIdEx(
 566     IN PEPROCESS Process
 567 );
 568
 569 /* marea.c *******************************************************************/
 570
 571 NTSTATUS
 572 NTAPI
 573 MmCreateMemoryArea(
 574     PMMSUPPORT AddressSpace,
 575     ULONG Type,
 576     PVOID *BaseAddress,
 577     SIZE_T Length,
 578     ULONG Protection,
 579     PMEMORY_AREA *Result,
 580     ULONG AllocationFlags,
 581     ULONG AllocationGranularity
 582 );
 583
 584 PMEMORY_AREA
 585 NTAPI
 586 MmLocateMemoryAreaByAddress(
 587     PMMSUPPORT AddressSpace,
 588     PVOID Address
 589 );
 590
 591 NTSTATUS
 592 NTAPI
 593 MmFreeMemoryArea(
 594     PMMSUPPORT AddressSpace,
 595     PMEMORY_AREA MemoryArea,
 596     PMM_FREE_PAGE_FUNC FreePage,
 597     PVOID FreePageContext
 598 );
 599
 600 VOID
 601 NTAPI
 602 MiRosCleanupMemoryArea(
 603     PEPROCESS Process,
 604     PMMVAD Vad);
 605
 606 PMEMORY_AREA
 607 NTAPI
 608 MmLocateMemoryAreaByRegion(
 609     PMMSUPPORT AddressSpace,
 610     PVOID Address,
 611     SIZE_T Length
 612 );
 613
 614 PVOID
 615 NTAPI
 616 MmFindGap(
 617     PMMSUPPORT AddressSpace,
 618     SIZE_T Length,
 619     ULONG_PTR Granularity,
 620     BOOLEAN TopDown
 621 );
 622
 623 VOID
 624 NTAPI
 625 MiRosCheckMemoryAreas(
 626    PMMSUPPORT AddressSpace);
 627
 628 VOID
 629 NTAPI
 630 MiCheckAllProcessMemoryAreas(VOID);
 631
 632 /* npool.c *******************************************************************/
 633
 634 CODE_SEG("INIT")
 635 VOID
 636 NTAPI
 637 MiInitializeNonPagedPool(VOID);
 638
 639 PVOID
 640 NTAPI
 641 MiAllocatePoolPages(
 642     IN POOL_TYPE PoolType,
 643     IN SIZE_T SizeInBytes
 644 );
 645
 646 POOL_TYPE
 647 NTAPI
 648 MmDeterminePoolType(
 649     IN PVOID VirtualAddress
 650 );
 651
 652 ULONG
 653 NTAPI
 654 MiFreePoolPages(
 655     IN PVOID StartingAddress
 656 );
 657
 658 /* pool.c *******************************************************************/
 659
 660 _Requires_lock_held_(PspQuotaLock)
 661 BOOLEAN
 662 NTAPI
 663 MmRaisePoolQuota(
 664     _In_ POOL_TYPE PoolType,
 665     _In_ SIZE_T CurrentMaxQuota,
 666     _Out_ PSIZE_T NewMaxQuota
 667 );
 668
 669 _Requires_lock_held_(PspQuotaLock)
 670 VOID
 671 NTAPI
 672 MmReturnPoolQuota(
 673     _In_ POOL_TYPE PoolType,
 674     _In_ SIZE_T QuotaToReturn
 675 );
 676
 677 /* mdl.c *********************************************************************/
 678
 679 VOID
 680 NTAPI
 681 MmBuildMdlFromPages(
 682     PMDL Mdl,
 683     PPFN_NUMBER Pages
 684 );
 685
 686 /* mminit.c ******************************************************************/
 687
 688 VOID
 689 NTAPI
 690 MmInit1(
 691     VOID
 692 );
 693
 694 CODE_SEG("INIT")
 695 BOOLEAN
 696 NTAPI
 697 MmInitSystem(IN ULONG Phase,
 698              IN PLOADER_PARAMETER_BLOCK LoaderBlock);
 699
 700
 701 /* pagefile.c ****************************************************************/
 702
 703 SWAPENTRY
 704 NTAPI
 705 MmAllocSwapPage(VOID);
 706
 707 VOID
 708 NTAPI
 709 MmFreeSwapPage(SWAPENTRY Entry);
 710
 711 CODE_SEG("INIT")
 712 VOID
 713 NTAPI
 714 MmInitPagingFile(VOID);
 715
 716 BOOLEAN
 717 NTAPI
 718 MmIsFileObjectAPagingFile(PFILE_OBJECT FileObject);
 719
 720 NTSTATUS
 721 NTAPI
 722 MmReadFromSwapPage(
 723     SWAPENTRY SwapEntry,
 724     PFN_NUMBER Page
 725 );
 726
 727 NTSTATUS
 728 NTAPI
 729 MmWriteToSwapPage(
 730     SWAPENTRY SwapEntry,
 731     PFN_NUMBER Page
 732 );
 733
 734 VOID
 735 NTAPI
 736 MmShowOutOfSpaceMessagePagingFile(VOID);
 737
 738 NTSTATUS
 739 NTAPI
 740 MiReadPageFile(
 741     _In_ PFN_NUMBER Page,
 742     _In_ ULONG PageFileIndex,
 743     _In_ ULONG_PTR PageFileOffset);
 744
 745 /* process.c ****************************************************************/
 746
 747 NTSTATUS
 748 NTAPI
 749 MmInitializeProcessAddressSpace(
 750     IN PEPROCESS Process,
 751     IN PEPROCESS Clone OPTIONAL,
 752     IN PVOID Section OPTIONAL,
 753     IN OUT PULONG Flags,
 754     IN POBJECT_NAME_INFORMATION *AuditName OPTIONAL
 755 );
 756
 757 NTSTATUS
 758 NTAPI
 759 MmCreatePeb(
 760     IN PEPROCESS Process,
 761     IN PINITIAL_PEB InitialPeb,
 762     OUT PPEB *BasePeb
 763 );
 764
 765 NTSTATUS
 766 NTAPI
 767 MmCreateTeb(
 768     IN PEPROCESS Process,
 769     IN PCLIENT_ID ClientId,
 770     IN PINITIAL_TEB InitialTeb,
 771     OUT PTEB* BaseTeb
 772 );
 773
 774 VOID
 775 NTAPI
 776 MmDeleteTeb(
 777     struct _EPROCESS *Process,
 778     PTEB Teb
 779 );
 780
 781 VOID
 782 NTAPI
 783 MmCleanProcessAddressSpace(IN PEPROCESS Process);
 784
 785 VOID
 786 NTAPI
 787 MmDeleteProcessAddressSpace(IN PEPROCESS Process);
 788
 789 ULONG
 790 NTAPI
 791 MmGetSessionLocaleId(VOID);
 792
 793 NTSTATUS
 794 NTAPI
 795 MmSetMemoryPriorityProcess(
 796     IN PEPROCESS Process,
 797     IN UCHAR MemoryPriority
 798 );
 799
 800 /* i386/pfault.c *************************************************************/
 801
 802 NTSTATUS
 803 NTAPI
 804 MmPageFault(
 805     ULONG Cs,
 806     PULONG Eip,
 807     PULONG Eax,
 808     ULONG Cr2,
 809     ULONG ErrorCode
 810 );
 811
 812 /* special.c *****************************************************************/
 813
 814 VOID
 815 NTAPI
 816 MiInitializeSpecialPool(VOID);
 817
 818 BOOLEAN
 819 NTAPI
 820 MmUseSpecialPool(
 821     IN SIZE_T NumberOfBytes,
 822     IN ULONG Tag);
 823
 824 BOOLEAN
 825 NTAPI
 826 MmIsSpecialPoolAddress(
 827     IN PVOID P);
 828
 829 BOOLEAN
 830 NTAPI
 831 MmIsSpecialPoolAddressFree(
 832     IN PVOID P);
 833
 834 PVOID
 835 NTAPI
 836 MmAllocateSpecialPool(
 837     IN SIZE_T NumberOfBytes,
 838     IN ULONG Tag,
 839     IN POOL_TYPE PoolType,
 840     IN ULONG SpecialType);
 841
 842 VOID
 843 NTAPI
 844 MmFreeSpecialPool(
 845     IN PVOID P);
 846
 847 /* mm.c **********************************************************************/
 848
 849 NTSTATUS
 850 NTAPI
 851 MmAccessFault(
 852     IN ULONG FaultCode,
 853     IN PVOID Address,
 854     IN KPROCESSOR_MODE Mode,
 855     IN PVOID TrapInformation
 856 );
 857
 858 /* process.c *****************************************************************/
 859
 860 PVOID
 861 NTAPI
 862 MmCreateKernelStack(BOOLEAN GuiStack, UCHAR Node);
 863
 864 VOID
 865 NTAPI
 866 MmDeleteKernelStack(PVOID Stack,
 867                     BOOLEAN GuiStack);
 868
 869 /* balance.c / pagefile.c******************************************************/
 870
 871 FORCEINLINE VOID UpdateTotalCommittedPages(LONG Delta)
 872 {
 873     /*
 874      * Add up all the used "Committed" memory + pagefile.
 875      * Not sure this is right. 8^\
 876      * MmTotalCommittedPages should be adjusted consistently with
 877      * other counters at different places.
 878      *
 879        MmTotalCommittedPages = MiMemoryConsumers[MC_SYSTEM].PagesUsed +
 880                                MiMemoryConsumers[MC_USER].PagesUsed +
 881                                MiUsedSwapPages;
 882      */
 883
 884     /* Update Commitment */
 885     SIZE_T TotalCommittedPages = InterlockedExchangeAddSizeT(&MmTotalCommittedPages, Delta) + Delta;
 886
 887     /* Update Peak = max(Peak, Total) in a lockless way */
 888     SIZE_T PeakCommitment = MmPeakCommitment;
 889     while (TotalCommittedPages > PeakCommitment &&
 890            InterlockedCompareExchangeSizeT(&MmPeakCommitment, TotalCommittedPages, PeakCommitment) != PeakCommitment)
 891     {
 892         PeakCommitment = MmPeakCommitment;
 893     }
 894 }
 895
 896 /* balance.c *****************************************************************/
 897
 898 CODE_SEG("INIT")
 899 VOID
 900 NTAPI
 901 MmInitializeMemoryConsumer(
 902     ULONG Consumer,
 903     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed)
 904 );
 905
 906 CODE_SEG("INIT")
 907 VOID
 908 NTAPI
 909 MmInitializeBalancer(
 910     ULONG NrAvailablePages,
 911     ULONG NrSystemPages
 912 );
 913
 914 NTSTATUS
 915 NTAPI
 916 MmReleasePageMemoryConsumer(
 917     ULONG Consumer,
 918     PFN_NUMBER Page
 919 );
 920
 921 NTSTATUS
 922 NTAPI
 923 MmRequestPageMemoryConsumer(
 924     ULONG Consumer,
 925     BOOLEAN MyWait,
 926     PPFN_NUMBER AllocatedPage
 927 );
 928
 929 CODE_SEG("INIT")
 930 VOID
 931 NTAPI
 932 MiInitBalancerThread(VOID);
 933
 934 VOID
 935 NTAPI
 936 MmRebalanceMemoryConsumers(VOID);
 937
 938 /* rmap.c **************************************************************/
 939 #define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff)
 940 #define RMAP_IS_SEGMENT(x) (((ULONG_PTR)(x) & RMAP_SEGMENT_MASK) == RMAP_SEGMENT_MASK)
 941
 942 VOID
 943 NTAPI
 944 MmSetRmapListHeadPage(
 945     PFN_NUMBER Page,
 946     struct _MM_RMAP_ENTRY* ListHead
 947 );
 948
 949 struct _MM_RMAP_ENTRY*
 950 NTAPI
 951 MmGetRmapListHeadPage(PFN_NUMBER Page);
 952
 953 VOID
 954 NTAPI
 955 MmInsertRmap(
 956     PFN_NUMBER Page,
 957     struct _EPROCESS *Process,
 958     PVOID Address
 959 );
 960
 961 VOID
 962 NTAPI
 963 MmDeleteAllRmaps(
 964     PFN_NUMBER Page,
 965     PVOID Context,
 966     VOID (*DeleteMapping)(PVOID Context, struct _EPROCESS *Process, PVOID Address)
 967 );
 968
 969 VOID
 970 NTAPI
 971 MmDeleteRmap(
 972     PFN_NUMBER Page,
 973     struct _EPROCESS *Process,
 974     PVOID Address
 975 );
 976
 977 CODE_SEG("INIT")
 978 VOID
 979 NTAPI
 980 MmInitializeRmapList(VOID);
 981
 982 NTSTATUS
 983 NTAPI
 984 MmPageOutPhysicalAddress(PFN_NUMBER Page);
 985
 986 PMM_SECTION_SEGMENT
 987 NTAPI
 988 MmGetSectionAssociation(PFN_NUMBER Page,
 989                         PLARGE_INTEGER Offset);
 990
 991 /* freelist.c **********************************************************/
 992 _IRQL_raises_(DISPATCH_LEVEL)
 993 _IRQL_requires_max_(DISPATCH_LEVEL)
 994 _Requires_lock_not_held_(MmPfnLock)
 995 _Acquires_lock_(MmPfnLock)
 996 _IRQL_saves_
 997 FORCEINLINE
 998 KIRQL
 999 MiAcquirePfnLock(VOID)
1000 {
1001     return KeAcquireQueuedSpinLock(LockQueuePfnLock);
1002 }
1003
1004 _Requires_lock_held_(MmPfnLock)
1005 _Releases_lock_(MmPfnLock)
1006 _IRQL_requires_(DISPATCH_LEVEL)
1007 FORCEINLINE
1008 VOID
1009 MiReleasePfnLock(
1010     _In_ _IRQL_restores_ KIRQL OldIrql)
1011 {
1012     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1013 }
1014
1015 _IRQL_requires_min_(DISPATCH_LEVEL)
1016 _Requires_lock_not_held_(MmPfnLock)
1017 _Acquires_lock_(MmPfnLock)
1018 FORCEINLINE
1019 VOID
1020 MiAcquirePfnLockAtDpcLevel(VOID)
1021 {
1022     PKSPIN_LOCK_QUEUE LockQueue;
1023
1024     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
1025     LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
1026     KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
1027 }
1028
1029 _Requires_lock_held_(MmPfnLock)
1030 _Releases_lock_(MmPfnLock)
1031 _IRQL_requires_min_(DISPATCH_LEVEL)
1032 FORCEINLINE
1033 VOID
1034 MiReleasePfnLockFromDpcLevel(VOID)
1035 {
1036     PKSPIN_LOCK_QUEUE LockQueue;
1037
1038     LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
1039     KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
1040     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
1041 }
1042
1043 #define MI_ASSERT_PFN_LOCK_HELD() NT_ASSERT((KeGetCurrentIrql() >= DISPATCH_LEVEL) && (MmPfnLock != 0))
1044
1045 FORCEINLINE
1046 PMMPFN
1047 MiGetPfnEntry(IN PFN_NUMBER Pfn)
1048 {
1049     PMMPFN Page;
1050     extern RTL_BITMAP MiPfnBitMap;
1051
1052     /* Make sure the PFN number is valid */
1053     if (Pfn > MmHighestPhysicalPage) return NULL;
1054
1055     /* Make sure this page actually has a PFN entry */
1056     if ((MiPfnBitMap.Buffer) && !(RtlTestBit(&MiPfnBitMap, (ULONG)Pfn))) return NULL;
1057
1058     /* Get the entry */
1059     Page = &MmPfnDatabase[Pfn];
1060
1061     /* Return it */
1062     return Page;
1063 };
1064
1065 FORCEINLINE
1066 PFN_NUMBER
1067 MiGetPfnEntryIndex(IN PMMPFN Pfn1)
1068 {
1069     //
1070     // This will return the Page Frame Number (PFN) from the MMPFN
1071     //
1072     return Pfn1 - MmPfnDatabase;
1073 }
1074
1075 PFN_NUMBER
1076 NTAPI
1077 MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast);
1078
1079 PFN_NUMBER
1080 NTAPI
1081 MmGetLRUFirstUserPage(VOID);
1082
1083 VOID
1084 NTAPI
1085 MmDumpArmPfnDatabase(
1086    IN BOOLEAN StatusOnly
1087 );
1088
1089 VOID
1090 NTAPI
1091 MmZeroPageThread(
1092     VOID
1093 );
1094
1095 /* hypermap.c *****************************************************************/
1096 PVOID
1097 NTAPI
1098 MiMapPageInHyperSpace(IN PEPROCESS Process,
1099                       IN PFN_NUMBER Page,
1100                       IN PKIRQL OldIrql);
1101
1102 VOID
1103 NTAPI
1104 MiUnmapPageInHyperSpace(IN PEPROCESS Process,
1105                         IN PVOID Address,
1106                         IN KIRQL OldIrql);
1107
1108 PVOID
1109 NTAPI
1110 MiMapPagesInZeroSpace(IN PMMPFN Pfn1,
1111                       IN PFN_NUMBER NumberOfPages);
1112
1113 VOID
1114 NTAPI
1115 MiUnmapPagesInZeroSpace(IN PVOID VirtualAddress,
1116                         IN PFN_NUMBER NumberOfPages);
1117
1118 /* i386/page.c *********************************************************/
1119
1120 NTSTATUS
1121 NTAPI
1122 MmCreateVirtualMapping(
1123     struct _EPROCESS* Process,
1124     PVOID Address,
1125     ULONG flProtect,
1126     PFN_NUMBER Page
1127 );
1128
1129 NTSTATUS
1130 NTAPI
1131 MmCreateVirtualMappingUnsafe(
1132     struct _EPROCESS* Process,
1133     PVOID Address,
1134     ULONG flProtect,
1135     PFN_NUMBER Page
1136 );
1137
1138 NTSTATUS
1139 NTAPI
1140 MmCreatePhysicalMapping(
1141     _Inout_opt_ PEPROCESS Process,
1142     _In_ PVOID Address,
1143     _In_ ULONG flProtect,
1144     _In_ PFN_NUMBER Page);
1145
1146 ULONG
1147 NTAPI
1148 MmGetPageProtect(
1149     struct _EPROCESS* Process,
1150     PVOID Address);
1151
1152 VOID
1153 NTAPI
1154 MmSetPageProtect(
1155     struct _EPROCESS* Process,
1156     PVOID Address,
1157     ULONG flProtect
1158 );
1159
1160 BOOLEAN
1161 NTAPI
1162 MmIsPagePresent(
1163     struct _EPROCESS* Process,
1164     PVOID Address
1165 );
1166
1167 BOOLEAN
1168 NTAPI
1169 MmIsDisabledPage(
1170     struct _EPROCESS* Process,
1171     PVOID Address
1172 );
1173
1174 CODE_SEG("INIT")
1175 VOID
1176 NTAPI
1177 MmInitGlobalKernelPageDirectory(VOID);
1178
1179 VOID
1180 NTAPI
1181 MmDeletePageFileMapping(
1182     struct _EPROCESS *Process,
1183     PVOID Address,
1184     SWAPENTRY* SwapEntry
1185 );
1186
1187 NTSTATUS
1188 NTAPI
1189 MmCreatePageFileMapping(
1190     struct _EPROCESS *Process,
1191     PVOID Address,
1192     SWAPENTRY SwapEntry
1193 );
1194
1195 VOID
1196 NTAPI
1197 MmGetPageFileMapping(
1198     PEPROCESS Process,
1199     PVOID Address,
1200     SWAPENTRY *SwapEntry);
1201
1202 BOOLEAN
1203 NTAPI
1204 MmIsPageSwapEntry(
1205     struct _EPROCESS *Process,
1206     PVOID Address
1207 );
1208
1209 PFN_NUMBER
1210 NTAPI
1211 MmAllocPage(
1212     ULONG Consumer
1213 );
1214
1215 VOID
1216 NTAPI
1217 MmDereferencePage(PFN_NUMBER Page);
1218
1219 VOID
1220 NTAPI
1221 MmReferencePage(PFN_NUMBER Page);
1222
1223 ULONG
1224 NTAPI
1225 MmGetReferenceCountPage(PFN_NUMBER Page);
1226
1227 BOOLEAN
1228 NTAPI
1229 MmIsPageInUse(PFN_NUMBER Page);
1230
1231 VOID
1232 NTAPI
1233 MmSetSavedSwapEntryPage(
1234     PFN_NUMBER Page,
1235     SWAPENTRY SavedSwapEntry);
1236
1237 SWAPENTRY
1238 NTAPI
1239 MmGetSavedSwapEntryPage(PFN_NUMBER Page);
1240
1241 VOID
1242 NTAPI
1243 MmSetCleanPage(
1244     struct _EPROCESS *Process,
1245     PVOID Address
1246 );
1247
1248 VOID
1249 NTAPI
1250 MmSetDirtyBit(PEPROCESS Process, PVOID Address, BOOLEAN Bit);
1251 #define MmSetCleanPage(__P, __A) MmSetDirtyBit(__P, __A, FALSE)
1252 #define MmSetDirtyPage(__P, __A) MmSetDirtyBit(__P, __A, TRUE)
1253
1254 VOID
1255 NTAPI
1256 MmDeletePageTable(
1257     struct _EPROCESS *Process,
1258     PVOID Address
1259 );
1260
1261 PFN_NUMBER
1262 NTAPI
1263 MmGetPfnForProcess(
1264     struct _EPROCESS *Process,
1265     PVOID Address
1266 );
1267
1268 BOOLEAN
1269 NTAPI
1270 MmCreateProcessAddressSpace(
1271     IN ULONG MinWs,
1272     IN PEPROCESS Dest,
1273     IN PULONG_PTR DirectoryTableBase
1274 );
1275
1276 CODE_SEG("INIT")
1277 NTSTATUS
1278 NTAPI
1279 MmInitializeHandBuiltProcess(
1280     IN PEPROCESS Process,
1281     IN PULONG_PTR DirectoryTableBase
1282 );
1283
1284 CODE_SEG("INIT")
1285 NTSTATUS
1286 NTAPI
1287 MmInitializeHandBuiltProcess2(
1288     IN PEPROCESS Process
1289 );
1290
1291 NTSTATUS
1292 NTAPI
1293 MmSetExecuteOptions(IN ULONG ExecuteOptions);
1294
1295 NTSTATUS
1296 NTAPI
1297 MmGetExecuteOptions(IN PULONG ExecuteOptions);
1298
1299 _Success_(return)
1300 BOOLEAN
1301 MmDeleteVirtualMapping(
1302     _Inout_opt_ PEPROCESS Process,
1303     _In_ PVOID Address,
1304     _Out_opt_ BOOLEAN* WasDirty,
1305     _Out_opt_ PPFN_NUMBER Page
1306 );
1307
1308 _Success_(return)
1309 BOOLEAN
1310 MmDeletePhysicalMapping(
1311     _Inout_opt_ PEPROCESS Process,
1312     _In_ PVOID Address,
1313     _Out_opt_ BOOLEAN * WasDirty,
1314     _Out_opt_ PPFN_NUMBER Page
1315 );
1316
1317 /* arch/procsup.c ************************************************************/
1318
1319 BOOLEAN
1320 MiArchCreateProcessAddressSpace(
1321     _In_ PEPROCESS Process,
1322     _In_ PULONG_PTR DirectoryTableBase);
1323
1324 /* wset.c ********************************************************************/
1325
1326 NTSTATUS
1327 MmTrimUserMemory(
1328     ULONG Target,
1329     ULONG Priority,
1330     PULONG NrFreedPages
1331 );
1332
1333 /* region.c ************************************************************/
1334
1335 NTSTATUS
1336 NTAPI
1337 MmAlterRegion(
1338     PMMSUPPORT AddressSpace,
1339     PVOID BaseAddress,
1340     PLIST_ENTRY RegionListHead,
1341     PVOID StartAddress,
1342     SIZE_T Length,
1343     ULONG NewType,
1344     ULONG NewProtect,
1345     PMM_ALTER_REGION_FUNC AlterFunc
1346 );
1347
1348 VOID
1349 NTAPI
1350 MmInitializeRegion(
1351     PLIST_ENTRY RegionListHead,
1352     SIZE_T Length,
1353     ULONG Type,
1354     ULONG Protect
1355 );
1356
1357 PMM_REGION
1358 NTAPI
1359 MmFindRegion(
1360     PVOID BaseAddress,
1361     PLIST_ENTRY RegionListHead,
1362     PVOID Address,
1363     PVOID* RegionBaseAddress
1364 );
1365
1366 /* section.c *****************************************************************/
1367
1368 #define PFN_FROM_SSE(E)          ((PFN_NUMBER)((E) >> PAGE_SHIFT))
1369 #define IS_SWAP_FROM_SSE(E)      ((E) & 0x00000001)
1370 #define MM_IS_WAIT_PTE(E)        \
1371     (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
1372 #define MAKE_PFN_SSE(P)          ((ULONG_PTR)((P) << PAGE_SHIFT))
1373 #define SWAPENTRY_FROM_SSE(E)    ((E) >> 1)
1374 #define MAKE_SWAP_SSE(S)         (((ULONG_PTR)(S) << 1) | 0x1)
1375 #define DIRTY_SSE(E)             ((E) | 2)
1376 #define CLEAN_SSE(E)             ((E) & ~2)
1377 #define IS_DIRTY_SSE(E)          ((E) & 2)
1378 #define WRITE_SSE(E)             ((E) | 4)
1379 #define IS_WRITE_SSE(E)          ((E) & 4)
1380 #ifdef _WIN64
1381 #define PAGE_FROM_SSE(E)         ((E) & 0xFFFFFFF000ULL)
1382 #else
1383 #define PAGE_FROM_SSE(E)         ((E) & 0xFFFFF000)
1384 #endif
1385 #define SHARE_COUNT_FROM_SSE(E)  (((E) & 0x00000FFC) >> 3)
1386 #define MAX_SHARE_COUNT          0x1FF
1387 #define MAKE_SSE(P, C)           ((ULONG_PTR)((P) | ((C) << 3)))
1388 #define BUMPREF_SSE(E)           (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) + 1) << 3) | ((E) & 0x7))
1389 #define DECREF_SSE(E)            (PAGE_FROM_SSE(E) | ((SHARE_COUNT_FROM_SSE(E) - 1) << 3) | ((E) & 0x7))
1390
1391 VOID
1392 NTAPI
1393 _MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
1394                       const char *file,
1395                       int line);
1396
1397 #define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
1398
1399 VOID
1400 NTAPI
1401 _MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
1402                         const char *file,
1403                         int line);
1404
1405 #define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
1406
1407 VOID
1408 NTAPI
1409 MmGetImageInformation(
1410     OUT PSECTION_IMAGE_INFORMATION ImageInformation
1411 );
1412
1413 PFILE_OBJECT
1414 NTAPI
1415 MmGetFileObjectForSection(
1416     IN PVOID Section
1417 );
1418 NTSTATUS
1419 NTAPI
1420 MmGetFileNameForAddress(
1421     IN PVOID Address,
1422     OUT PUNICODE_STRING ModuleName
1423 );
1424
1425 NTSTATUS
1426 NTAPI
1427 MmGetFileNameForSection(
1428     IN PVOID Section,
1429     OUT POBJECT_NAME_INFORMATION *ModuleName
1430 );
1431
1432 NTSTATUS
1433 NTAPI
1434 MmQuerySectionView(
1435     PMEMORY_AREA MemoryArea,
1436     PVOID Address,
1437     PMEMORY_BASIC_INFORMATION Info,
1438     PSIZE_T ResultLength
1439 );
1440
1441 NTSTATUS
1442 NTAPI
1443 MmProtectSectionView(
1444     PMMSUPPORT AddressSpace,
1445     PMEMORY_AREA MemoryArea,
1446     PVOID BaseAddress,
1447     SIZE_T Length,
1448     ULONG Protect,
1449     PULONG OldProtect
1450 );
1451
1452 CODE_SEG("INIT")
1453 NTSTATUS
1454 NTAPI
1455 MmInitSectionImplementation(VOID);
1456
1457 NTSTATUS
1458 NTAPI
1459 MmNotPresentFaultSectionView(
1460     PMMSUPPORT AddressSpace,
1461     MEMORY_AREA* MemoryArea,
1462     PVOID Address,
1463     BOOLEAN Locked
1464 );
1465
1466 NTSTATUS
1467 NTAPI
1468 MmPageOutSectionView(
1469     PMMSUPPORT AddressSpace,
1470     PMEMORY_AREA MemoryArea,
1471     PVOID Address,
1472     ULONG_PTR Entry
1473 );
1474
1475 CODE_SEG("INIT")
1476 NTSTATUS
1477 NTAPI
1478 MmCreatePhysicalMemorySection(VOID);
1479
1480 NTSTATUS
1481 NTAPI
1482 MmAccessFaultSectionView(
1483     PMMSUPPORT AddressSpace,
1484     MEMORY_AREA* MemoryArea,
1485     PVOID Address,
1486     BOOLEAN Locked
1487 );
1488
1489 VOID
1490 NTAPI
1491 MmFreeSectionSegments(PFILE_OBJECT FileObject);
1492
1493 /* Exported from NT 6.2 onward. We keep it internal. */
1494 NTSTATUS
1495 NTAPI
1496 MmMapViewInSystemSpaceEx(
1497     _In_ PVOID Section,
1498     _Outptr_result_bytebuffer_ (*ViewSize) PVOID *MappedBase,
1499     _Inout_ PSIZE_T ViewSize,
1500     _Inout_ PLARGE_INTEGER SectionOffset,
1501     _In_ ULONG_PTR Flags);
1502
1503 BOOLEAN
1504 NTAPI
1505 MmArePagesResident(
1506     _In_ PEPROCESS Process,
1507     _In_ PVOID BaseAddress,
1508     _In_ ULONG Length);
1509
1510 NTSTATUS
1511 NTAPI
1512 MmMakePagesDirty(
1513     _In_ PEPROCESS Process,
1514     _In_ PVOID Address,
1515     _In_ ULONG Length);
1516
1517 NTSTATUS
1518 NTAPI
1519 MmFlushSegment(
1520     _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
1521     _In_opt_ PLARGE_INTEGER Offset,
1522     _In_ ULONG Length,
1523     _Out_opt_ PIO_STATUS_BLOCK Iosb);
1524
1525 NTSTATUS
1526 NTAPI
1527 MmMakeDataSectionResident(
1528     _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
1529     _In_ LONGLONG Offset,
1530     _In_ ULONG Length,
1531     _In_ PLARGE_INTEGER ValidDataLength);
1532
1533 BOOLEAN
1534 NTAPI
1535 MmPurgeSegment(
1536     _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
1537     _In_opt_ PLARGE_INTEGER Offset,
1538     _In_ ULONG Length);
1539
1540 BOOLEAN
1541 NTAPI
1542 MmCheckDirtySegment(
1543     PMM_SECTION_SEGMENT Segment,
1544     PLARGE_INTEGER Offset,
1545     BOOLEAN ForceDirty,
1546     BOOLEAN PageOut);
1547
1548 BOOLEAN
1549 NTAPI
1550 MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea,
1551                                  PMM_SECTION_SEGMENT Segment,
1552                                  PLARGE_INTEGER Offset,
1553                                  BOOLEAN Dirty,
1554                                  BOOLEAN PageOut,
1555                                  ULONG_PTR *InEntry);
1556
1557 _When_(OldIrql == MM_NOIRQL, _IRQL_requires_max_(DISPATCH_LEVEL))
1558 _When_(OldIrql == MM_NOIRQL, _Requires_lock_not_held_(MmPfnLock))
1559 _When_(OldIrql != MM_NOIRQL, _Requires_lock_held_(MmPfnLock))
1560 _When_(OldIrql != MM_NOIRQL, _Releases_lock_(MmPfnLock))
1561 _When_(OldIrql != MM_NOIRQL, _IRQL_requires_(DISPATCH_LEVEL))
1562 VOID
1563 NTAPI
1564 MmDereferenceSegmentWithLock(
1565     _In_ PMM_SECTION_SEGMENT Segment,
1566     _In_ _When_(OldIrql != MM_NOIRQL, _IRQL_restores_) KIRQL OldIrql);
1567
1568 _IRQL_requires_max_(DISPATCH_LEVEL)
1569 _Requires_lock_not_held_(MmPfnLock)
1570 FORCEINLINE
1571 VOID
1572 MmDereferenceSegment(PMM_SECTION_SEGMENT Segment)
1573 {
1574     MmDereferenceSegmentWithLock(Segment, MM_NOIRQL);
1575 }
1576
1577 NTSTATUS
1578 NTAPI
1579 MmExtendSection(
1580     _In_ PVOID Section,
1581     _Inout_ PLARGE_INTEGER NewSize);
1582
1583 /* sptab.c *******************************************************************/
1584
1585 NTSTATUS
1586 NTAPI
1587 _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
1588                               PLARGE_INTEGER Offset,
1589                               ULONG_PTR Entry,
1590                               const char *file,
1591                               int line);
1592
1593 ULONG_PTR
1594 NTAPI
1595 _MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
1596                               PLARGE_INTEGER Offset,
1597                               const char *file,
1598                               int line);
1599
1600 #define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
1601
1602 #define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
1603
1604 /* sysldr.c ******************************************************************/
1605
1606 CODE_SEG("INIT")
1607 VOID
1608 NTAPI
1609 MiReloadBootLoadedDrivers(
1610     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1611 );
1612
1613 CODE_SEG("INIT")
1614 BOOLEAN
1615 NTAPI
1616 MiInitializeLoadedModuleList(
1617     IN PLOADER_PARAMETER_BLOCK LoaderBlock
1618 );
1619
1620 BOOLEAN
1621 NTAPI
1622 MmChangeKernelResourceSectionProtection(IN ULONG_PTR ProtectionMask);
1623
1624 VOID
1625 NTAPI
1626 MmMakeKernelResourceSectionWritable(VOID);
1627
1628 NTSTATUS
1629 NTAPI
1630 MmLoadSystemImage(
1631     IN PUNICODE_STRING FileName,
1632     IN PUNICODE_STRING NamePrefix OPTIONAL,
1633     IN PUNICODE_STRING LoadedName OPTIONAL,
1634     IN ULONG Flags,
1635     OUT PVOID *ModuleObject,
1636     OUT PVOID *ImageBaseAddress
1637 );
1638
1639 NTSTATUS
1640 NTAPI
1641 MmUnloadSystemImage(
1642     IN PVOID ImageHandle
1643 );
1644
1645 NTSTATUS
1646 NTAPI
1647 MmCheckSystemImage(
1648     IN HANDLE ImageHandle,
1649     IN BOOLEAN PurgeSection
1650 );
1651
1652 NTSTATUS
1653 NTAPI
1654 MmCallDllInitialize(
1655     _In_ PLDR_DATA_TABLE_ENTRY LdrEntry,
1656     _In_ PLIST_ENTRY ModuleListHead);
1657
1658 VOID
1659 NTAPI
1660 MmFreeDriverInitialization(
1661     IN PLDR_DATA_TABLE_ENTRY LdrEntry);
1662
1663 /* ReactOS-only, used by psmgr.c PspLookupSystemDllEntryPoint() as well */
1664 NTSTATUS
1665 NTAPI
1666 RtlpFindExportedRoutineByName(
1667     _In_ PVOID ImageBase,
1668     _In_ PCSTR ExportName,
1669     _Out_ PVOID* Function,
1670     _Out_opt_ PBOOLEAN IsForwarder,
1671     _In_ NTSTATUS NotFoundStatus);
1672
1673 /* Exported from NT 10.0 onward. We keep it internal. */
1674 PVOID
1675 NTAPI
1676 RtlFindExportedRoutineByName(
1677     _In_ PVOID ImageBase,
1678     _In_ PCSTR ExportName);
1679
1680 /* procsup.c *****************************************************************/
1681
1682 NTSTATUS
1683 NTAPI
1684 MmGrowKernelStack(
1685     IN PVOID StackPointer
1686 );
1687
1688
1689 FORCEINLINE
1690 VOID
1691 MmLockAddressSpace(PMMSUPPORT AddressSpace)
1692 {
1693     ASSERT(!PsGetCurrentThread()->OwnsProcessWorkingSetExclusive);
1694     ASSERT(!PsGetCurrentThread()->OwnsProcessWorkingSetShared);
1695     ASSERT(!PsGetCurrentThread()->OwnsSystemWorkingSetExclusive);
1696     ASSERT(!PsGetCurrentThread()->OwnsSystemWorkingSetShared);
1697     ASSERT(!PsGetCurrentThread()->OwnsSessionWorkingSetExclusive);
1698     ASSERT(!PsGetCurrentThread()->OwnsSessionWorkingSetShared);
1699     KeAcquireGuardedMutex(&CONTAINING_RECORD(AddressSpace, EPROCESS, Vm)->AddressCreationLock);
1700 }
1701
1702 FORCEINLINE
1703 VOID
1704 MmUnlockAddressSpace(PMMSUPPORT AddressSpace)
1705 {
1706     KeReleaseGuardedMutex(&CONTAINING_RECORD(AddressSpace, EPROCESS, Vm)->AddressCreationLock);
1707 }
1708
1709 FORCEINLINE
1710 PEPROCESS
1711 MmGetAddressSpaceOwner(IN PMMSUPPORT AddressSpace)
1712 {
1713     if (AddressSpace == MmKernelAddressSpace) return NULL;
1714     return CONTAINING_RECORD(AddressSpace, EPROCESS, Vm);
1715 }
1716
1717 FORCEINLINE
1718 PMMSUPPORT
1719 MmGetCurrentAddressSpace(VOID)
1720 {
1721     return &((PEPROCESS)KeGetCurrentThread()->ApcState.Process)->Vm;
1722 }
1723
1724 FORCEINLINE
1725 PMMSUPPORT
1726 MmGetKernelAddressSpace(VOID)
1727 {
1728     return MmKernelAddressSpace;
1729 }
1730
1731
1732 /* expool.c ******************************************************************/
1733
1734 VOID
1735 NTAPI
1736 ExpCheckPoolAllocation(
1737     PVOID P,
1738     POOL_TYPE PoolType,
1739     ULONG Tag);
1740
1741 VOID
1742 NTAPI
1743 ExReturnPoolQuota(
1744     IN PVOID P);
1745
1746
1747 /* mmsup.c *****************************************************************/
1748
1749 NTSTATUS
1750 NTAPI
1751 MmAdjustWorkingSetSize(
1752     IN SIZE_T WorkingSetMinimumInBytes,
1753     IN SIZE_T WorkingSetMaximumInBytes,
1754     IN ULONG SystemCache,
1755     IN BOOLEAN IncreaseOkay);
1756
1757
1758 /* session.c *****************************************************************/
1759
1760 _IRQL_requires_max_(APC_LEVEL)
1761 NTSTATUS
1762 NTAPI
1763 MmAttachSession(
1764     _Inout_ PVOID SessionEntry,
1765     _Out_ PKAPC_STATE ApcState);
1766
1767 _IRQL_requires_max_(APC_LEVEL)
1768 VOID
1769 NTAPI
1770 MmDetachSession(
1771     _Inout_ PVOID SessionEntry,
1772     _Out_ PKAPC_STATE ApcState);
1773
1774 VOID
1775 NTAPI
1776 MmQuitNextSession(
1777     _Inout_ PVOID SessionEntry);
1778
1779 PVOID
1780 NTAPI
1781 MmGetSessionById(
1782     _In_ ULONG SessionId);
1783
1784 _IRQL_requires_max_(APC_LEVEL)
1785 VOID
1786 NTAPI
1787 MmSetSessionLocaleId(
1788     _In_ LCID LocaleId);
1789
1790 /* shutdown.c *****************************************************************/
1791
1792 VOID
1793 MmShutdownSystem(IN ULONG Phase);
1794
1795 /* virtual.c *****************************************************************/
1796
1797 NTSTATUS
1798 NTAPI
1799 MmCopyVirtualMemory(IN PEPROCESS SourceProcess,
1800                     IN PVOID SourceAddress,
1801                     IN PEPROCESS TargetProcess,
1802                     OUT PVOID TargetAddress,
1803                     IN SIZE_T BufferSize,
1804                     IN KPROCESSOR_MODE PreviousMode,
1805                     OUT PSIZE_T ReturnSize);
1806
1807 /* wslist.cpp ****************************************************************/
1808 _Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex)
1809 VOID
1810 NTAPI
1811 MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet);
1812
1813 #ifdef __cplusplus
1814 } // extern "C"
1815
1816 namespace ntoskrnl
1817 {
1818 using MiPfnLockGuard = const KiQueuedSpinLockGuard<LockQueuePfnLock>;
1819 } // namespace ntoskrnl
1820
1821 #endif