2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/miarm.h
5 * PURPOSE: ARM Memory Manager Header
6 * PROGRAMMERS: ReactOS Portable Systems Group
11 #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS
13 /* Make the code cleaner with some definitions for size multiples */
15 #define _1MB (1024 * _1KB)
16 #define _1GB (1024 * _1MB)
18 /* Everyone loves 64K */
19 #define _64K (64 * _1KB)
21 /* Area mapped by a PDE */
22 #define PDE_MAPPED_VA (PTE_COUNT * PAGE_SIZE)
24 /* Size of a page table */
25 #define PT_SIZE (PTE_COUNT * sizeof(MMPTE))
27 /* Size of a page directory */
28 #define PD_SIZE (PDE_COUNT * sizeof(MMPDE))
30 /* Stop using these! */
31 #define PD_COUNT PPE_PER_PAGE
32 #define PDE_COUNT PDE_PER_PAGE
33 #define PTE_COUNT PTE_PER_PAGE
35 /* Size of all page directories for a process */
36 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
38 C_ASSERT(SYSTEM_PD_SIZE
== PAGE_SIZE
);
42 // Protection Bits part of the internal memory manager Protection Mask, from:
43 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
44 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
45 // and public assertions.
47 #define MM_ZERO_ACCESS 0
50 #define MM_EXECUTE_READ 3
51 #define MM_READWRITE 4
52 #define MM_WRITECOPY 5
53 #define MM_EXECUTE_READWRITE 6
54 #define MM_EXECUTE_WRITECOPY 7
55 #define MM_PROTECT_ACCESS 7
58 // These are flags on top of the actual protection mask
60 #define MM_NOCACHE 0x08
61 #define MM_GUARDPAGE 0x10
62 #define MM_WRITECOMBINE 0x18
63 #define MM_PROTECT_SPECIAL 0x18
66 // These are special cases
68 #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE)
69 #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE)
70 #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
71 #define MM_INVALID_PROTECTION 0xFFFFFFFF
74 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
75 // The Memory Manager's definition define the attributes that must be preserved
76 // and these PTE definitions describe the attributes in the hardware sense. This
77 // helps deal with hardware differences between the actual boolean expression of
80 // For example, in the logical attributes, we want to express read-only as a flag
81 // but on x86, it is writability that must be set. On the other hand, on x86, just
82 // like in the kernel, it is disabling the caches that requires a special flag,
83 // while on certain architectures such as ARM, it is enabling the cache which
86 #if defined(_M_IX86) || defined(_M_AMD64)
90 #define PTE_READONLY 0 // Doesn't exist on x86
91 #define PTE_EXECUTE 0 // Not worrying about NX yet
92 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
93 #define PTE_READWRITE 0x2
94 #define PTE_WRITECOPY 0x200
95 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet
96 #define PTE_EXECUTE_WRITECOPY 0x200
97 #define PTE_PROTOTYPE 0x400
102 #define PTE_VALID 0x1
103 #define PTE_ACCESSED 0x20
104 #define PTE_DIRTY 0x40
109 #define PTE_ENABLE_CACHE 0
110 #define PTE_DISABLE_CACHE 0x10
111 #define PTE_WRITECOMBINED_CACHE 0x10
112 #elif defined(_M_ARM)
113 #define PTE_READONLY 0x200
114 #define PTE_EXECUTE 0 // Not worrying about NX yet
115 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
116 #define PTE_READWRITE 0 // Doesn't exist on ARM
117 #define PTE_WRITECOPY 0 // Doesn't exist on ARM
118 #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet
119 #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet
120 #define PTE_PROTOTYPE 0x400 // Using the Shared bit
124 #define PTE_ENABLE_CACHE 0
125 #define PTE_DISABLE_CACHE 0x10
126 #define PTE_WRITECOMBINED_CACHE 0x10
128 #error Define these please!
131 extern const ULONG_PTR MmProtectToPteMask
[32];
132 extern const ULONG MmProtectToValue
[32];
135 // Assertions for session images, addresses, and PTEs
137 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
138 (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
140 #define MI_IS_SESSION_ADDRESS(Address) \
141 (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
143 #define MI_IS_SESSION_PTE(Pte) \
144 ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
146 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
147 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
149 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
150 (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
152 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
153 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
156 // Creates a software PTE with the given protection
158 #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
161 // Marks a PTE as deleted
163 #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
164 #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1)
167 // Special values for LoadedImports
170 #define MM_SYSLDR_NO_IMPORTS (PVOID)0xFFFFFFFFFFFFFFFEULL
171 #define MM_SYSLDR_BOOT_LOADED (PVOID)0xFFFFFFFFFFFFFFFFULL
173 #define MM_SYSLDR_NO_IMPORTS (PVOID)0xFFFFFFFE
174 #define MM_SYSLDR_BOOT_LOADED (PVOID)0xFFFFFFFF
176 #define MM_SYSLDR_SINGLE_ENTRY 0x1
179 // Number of initial session IDs
181 #define MI_INITIAL_SESSION_IDS 64
183 #if defined(_M_IX86) || defined(_M_ARM)
187 #define LIST_HEAD 0xFFFFFFFF
190 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
191 // we need a manual definition suited to the number of bits in the PteFrame.
192 // This is used as a LIST_HEAD for the colored list
194 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
195 #elif defined(_M_AMD64)
196 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
197 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
199 #error Define these please!
203 // Special IRQL value (found in assertions)
205 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
208 // Returns the color of a page
210 #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)
211 #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor))
212 #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
215 // Prototype PTEs that don't yet have a pagefile association
218 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
220 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
224 // Number of session data and tag pages
226 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
227 #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
230 // Used by MiCheckSecuredVad
232 #define MM_READ_WRITE_ALLOWED 11
233 #define MM_READ_ONLY_ALLOWED 10
234 #define MM_NO_ACCESS_ALLOWED 01
235 #define MM_DELETE_CHECK 85
238 // System views are binned into 64K chunks
240 #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K
243 // FIXFIX: These should go in ex.h after the pool merge
246 #define POOL_BLOCK_SIZE 16
248 #define POOL_BLOCK_SIZE 8
250 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
251 #define BASE_POOL_TYPE_MASK 1
252 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
255 // Pool debugging/analysis/tracing flags
257 #define POOL_FLAG_CHECK_TIMERS 0x1
258 #define POOL_FLAG_CHECK_WORKERS 0x2
259 #define POOL_FLAG_CHECK_RESOURCES 0x4
260 #define POOL_FLAG_VERIFIER 0x8
261 #define POOL_FLAG_CHECK_DEADLOCK 0x10
262 #define POOL_FLAG_SPECIAL_POOL 0x20
263 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
264 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
267 // BAD_POOL_HEADER codes during pool bugcheck
269 #define POOL_CORRUPTED_LIST 3
270 #define POOL_SIZE_OR_INDEX_MISMATCH 5
271 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
272 #define POOL_HEADER_NOT_ALIGNED 7
273 #define POOL_HEADER_IS_ZERO 8
274 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
275 #define POOL_ENTRY_NOT_FOUND 10
278 // BAD_POOL_CALLER codes during pool bugcheck
280 #define POOL_ENTRY_CORRUPTED 1
281 #define POOL_ENTRY_ALREADY_FREE 6
282 #define POOL_ENTRY_NOT_ALLOCATED 7
283 #define POOL_ALLOC_IRQL_INVALID 8
284 #define POOL_FREE_IRQL_INVALID 9
285 #define POOL_BILLED_PROCESS_INVALID 13
286 #define POOL_HEADER_SIZE_INVALID 32
288 typedef struct _POOL_DESCRIPTOR
293 ULONG RunningDeAllocs
;
299 LONG PendingFreeDepth
;
302 LIST_ENTRY ListHeads
[POOL_LISTS_PER_PAGE
];
303 } POOL_DESCRIPTOR
, *PPOOL_DESCRIPTOR
;
305 typedef struct _POOL_HEADER
312 USHORT PreviousSize
:8;
317 USHORT PreviousSize
:9;
331 PEPROCESS ProcessBilled
;
337 USHORT AllocatorBackTraceIndex
;
341 } POOL_HEADER
, *PPOOL_HEADER
;
343 C_ASSERT(sizeof(POOL_HEADER
) == POOL_BLOCK_SIZE
);
344 C_ASSERT(POOL_BLOCK_SIZE
== sizeof(LIST_ENTRY
));
346 typedef struct _POOL_TRACKER_TABLE
351 SIZE_T NonPagedBytes
;
355 } POOL_TRACKER_TABLE
, *PPOOL_TRACKER_TABLE
;
357 typedef struct _POOL_TRACKER_BIG_PAGES
363 } POOL_TRACKER_BIG_PAGES
, *PPOOL_TRACKER_BIG_PAGES
;
365 extern ULONG ExpNumberOfPagedPools
;
366 extern POOL_DESCRIPTOR NonPagedPoolDescriptor
;
367 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor
[16 + 1];
368 extern PPOOL_TRACKER_TABLE PoolTrackTable
;
374 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
377 UNICODE_STRING BaseName
;
378 } MI_LARGE_PAGE_DRIVER_ENTRY
, *PMI_LARGE_PAGE_DRIVER_ENTRY
;
380 typedef enum _MMSYSTEM_PTE_POOL_TYPE
383 NonPagedPoolExpansion
,
385 } MMSYSTEM_PTE_POOL_TYPE
;
387 typedef enum _MI_PFN_CACHE_ATTRIBUTE
393 } MI_PFN_CACHE_ATTRIBUTE
, *PMI_PFN_CACHE_ATTRIBUTE
;
395 typedef struct _PHYSICAL_MEMORY_RUN
398 PFN_NUMBER PageCount
;
399 } PHYSICAL_MEMORY_RUN
, *PPHYSICAL_MEMORY_RUN
;
401 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
404 PFN_NUMBER NumberOfPages
;
405 PHYSICAL_MEMORY_RUN Run
[1];
406 } PHYSICAL_MEMORY_DESCRIPTOR
, *PPHYSICAL_MEMORY_DESCRIPTOR
;
408 typedef struct _MMCOLOR_TABLES
413 } MMCOLOR_TABLES
, *PMMCOLOR_TABLES
;
415 typedef struct _MI_LARGE_PAGE_RANGES
417 PFN_NUMBER StartFrame
;
418 PFN_NUMBER LastFrame
;
419 } MI_LARGE_PAGE_RANGES
, *PMI_LARGE_PAGE_RANGES
;
421 typedef struct _MMVIEW
424 PCONTROL_AREA ControlArea
;
427 typedef struct _MMSESSION
429 KGUARDED_MUTEX SystemSpaceViewLock
;
430 PKGUARDED_MUTEX SystemSpaceViewLockPointer
;
431 PCHAR SystemSpaceViewStart
;
432 PMMVIEW SystemSpaceViewTable
;
433 ULONG SystemSpaceHashSize
;
434 ULONG SystemSpaceHashEntries
;
435 ULONG SystemSpaceHashKey
;
436 ULONG BitmapFailures
;
437 PRTL_BITMAP SystemSpaceBitMap
;
438 } MMSESSION
, *PMMSESSION
;
440 typedef struct _MM_SESSION_SPACE_FLAGS
443 ULONG DeletePending
:1;
445 } MM_SESSION_SPACE_FLAGS
;
447 typedef struct _MM_SESSION_SPACE
449 struct _MM_SESSION_SPACE
*GlobalVirtualAddress
;
454 MM_SESSION_SPACE_FLAGS Flags
;
457 LIST_ENTRY ProcessList
;
458 LARGE_INTEGER LastProcessSwappedOutTime
;
459 PFN_NUMBER SessionPageDirectoryIndex
;
460 SIZE_T NonPageablePages
;
461 SIZE_T CommittedPages
;
462 PVOID PagedPoolStart
;
464 PMMPDE PagedPoolBasePde
;
466 LONG ResidentProcessCount
;
467 ULONG SessionPoolAllocationFailures
[4];
468 LIST_ENTRY ImageList
;
472 PEPROCESS LastProcess
;
473 LONG ProcessReferenceToSession
;
474 LIST_ENTRY WsListEntry
;
475 GENERAL_LOOKASIDE Lookaside
[SESSION_POOL_LOOKASIDES
];
477 KGUARDED_MUTEX PagedPoolMutex
;
478 MM_PAGED_POOL_INFO PagedPoolInfo
;
481 PDRIVER_UNLOAD Win32KDriverUnload
;
482 POOL_DESCRIPTOR PagedPool
;
483 #if defined (_M_AMD64)
488 #if defined (_M_AMD64)
489 PMMPTE SpecialPoolFirstPte
;
490 PMMPTE SpecialPoolLastPte
;
491 PMMPTE NextPdeForSpecialPoolExpansion
;
492 PMMPTE LastPdeForSpecialPoolExpansion
;
493 PFN_NUMBER SpecialPagesInUse
;
495 LONG ImageLoadingCount
;
496 } MM_SESSION_SPACE
, *PMM_SESSION_SPACE
;
498 extern PMM_SESSION_SPACE MmSessionSpace
;
499 extern MMPTE HyperTemplatePte
;
500 extern MMPDE ValidKernelPde
;
501 extern MMPTE ValidKernelPte
;
502 extern MMPDE ValidKernelPdeLocal
;
503 extern MMPTE ValidKernelPteLocal
;
504 extern MMPDE DemandZeroPde
;
505 extern MMPTE DemandZeroPte
;
506 extern MMPTE PrototypePte
;
507 extern MMPTE MmDecommittedPte
;
508 extern BOOLEAN MmLargeSystemCache
;
509 extern BOOLEAN MmZeroPageFile
;
510 extern BOOLEAN MmProtectFreedNonPagedPool
;
511 extern BOOLEAN MmTrackLockedPages
;
512 extern BOOLEAN MmTrackPtes
;
513 extern BOOLEAN MmDynamicPfn
;
514 extern BOOLEAN MmMirroring
;
515 extern BOOLEAN MmMakeLowMemory
;
516 extern BOOLEAN MmEnforceWriteProtection
;
517 extern SIZE_T MmAllocationFragment
;
518 extern ULONG MmConsumedPoolPercentage
;
519 extern ULONG MmVerifyDriverBufferType
;
520 extern ULONG MmVerifyDriverLevel
;
521 extern WCHAR MmVerifyDriverBuffer
[512];
522 extern WCHAR MmLargePageDriverBuffer
[512];
523 extern LIST_ENTRY MiLargePageDriverList
;
524 extern BOOLEAN MiLargePageAllDrivers
;
525 extern ULONG MmVerifyDriverBufferLength
;
526 extern ULONG MmLargePageDriverBufferLength
;
527 extern SIZE_T MmSizeOfNonPagedPoolInBytes
;
528 extern SIZE_T MmMaximumNonPagedPoolInBytes
;
529 extern PFN_NUMBER MmMaximumNonPagedPoolInPages
;
530 extern PFN_NUMBER MmSizeOfPagedPoolInPages
;
531 extern PVOID MmNonPagedSystemStart
;
532 extern PVOID MmNonPagedPoolStart
;
533 extern PVOID MmNonPagedPoolExpansionStart
;
534 extern PVOID MmNonPagedPoolEnd
;
535 extern SIZE_T MmSizeOfPagedPoolInBytes
;
536 extern PVOID MmPagedPoolStart
;
537 extern PVOID MmPagedPoolEnd
;
538 extern PVOID MmSessionBase
;
539 extern SIZE_T MmSessionSize
;
540 extern PMMPTE MmFirstReservedMappingPte
, MmLastReservedMappingPte
;
541 extern PMMPTE MiFirstReservedZeroingPte
;
542 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes
[2][MmMaximumCacheType
];
543 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock
;
544 extern SIZE_T MmBootImageSize
;
545 extern PMMPTE MmSystemPtesStart
[MaximumPtePoolTypes
];
546 extern PMMPTE MmSystemPtesEnd
[MaximumPtePoolTypes
];
547 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor
;
548 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor
;
549 extern ULONG_PTR MxPfnAllocation
;
550 extern MM_PAGED_POOL_INFO MmPagedPoolInfo
;
551 extern RTL_BITMAP MiPfnBitMap
;
552 extern KGUARDED_MUTEX MmPagedPoolMutex
;
553 extern KGUARDED_MUTEX MmSectionCommitMutex
;
554 extern PVOID MmPagedPoolStart
;
555 extern PVOID MmPagedPoolEnd
;
556 extern PVOID MmNonPagedSystemStart
;
557 extern PVOID MiSystemViewStart
;
558 extern SIZE_T MmSystemViewSize
;
559 extern PVOID MmSessionBase
;
560 extern PVOID MiSessionSpaceEnd
;
561 extern PMMPTE MiSessionImagePteStart
;
562 extern PMMPTE MiSessionImagePteEnd
;
563 extern PMMPTE MiSessionBasePte
;
564 extern PMMPTE MiSessionLastPte
;
565 extern SIZE_T MmSizeOfPagedPoolInBytes
;
566 extern PMMPDE MmSystemPagePtes
;
567 extern PVOID MmSystemCacheStart
;
568 extern PVOID MmSystemCacheEnd
;
569 extern MMSUPPORT MmSystemCacheWs
;
570 extern SIZE_T MmAllocatedNonPagedPool
;
571 extern ULONG MmSpecialPoolTag
;
572 extern PVOID MmHyperSpaceEnd
;
573 extern PMMWSL MmSystemCacheWorkingSetList
;
574 extern SIZE_T MmMinimumNonPagedPoolSize
;
575 extern ULONG MmMinAdditionNonPagedPoolPerMb
;
576 extern SIZE_T MmDefaultMaximumNonPagedPool
;
577 extern ULONG MmMaxAdditionNonPagedPoolPerMb
;
578 extern ULONG MmSecondaryColors
;
579 extern ULONG MmSecondaryColorMask
;
580 extern ULONG MmNumberOfSystemPtes
;
581 extern ULONG MmMaximumNonPagedPoolPercent
;
582 extern ULONG MmLargeStackSize
;
583 extern PMMCOLOR_TABLES MmFreePagesByColor
[FreePageList
+ 1];
584 extern MMPFNLIST MmStandbyPageListByPriority
[8];
585 extern ULONG MmProductType
;
586 extern MM_SYSTEMSIZE MmSystemSize
;
587 extern PKEVENT MiLowMemoryEvent
;
588 extern PKEVENT MiHighMemoryEvent
;
589 extern PKEVENT MiLowPagedPoolEvent
;
590 extern PKEVENT MiHighPagedPoolEvent
;
591 extern PKEVENT MiLowNonPagedPoolEvent
;
592 extern PKEVENT MiHighNonPagedPoolEvent
;
593 extern PFN_NUMBER MmLowMemoryThreshold
;
594 extern PFN_NUMBER MmHighMemoryThreshold
;
595 extern PFN_NUMBER MiLowPagedPoolThreshold
;
596 extern PFN_NUMBER MiHighPagedPoolThreshold
;
597 extern PFN_NUMBER MiLowNonPagedPoolThreshold
;
598 extern PFN_NUMBER MiHighNonPagedPoolThreshold
;
599 extern PFN_NUMBER MmMinimumFreePages
;
600 extern PFN_NUMBER MmPlentyFreePages
;
601 extern SIZE_T MmMinimumStackCommitInBytes
;
602 extern PFN_COUNT MiExpansionPoolPagesInitialCharge
;
603 extern PFN_NUMBER MmResidentAvailablePages
;
604 extern PFN_NUMBER MmResidentAvailableAtInit
;
605 extern ULONG MmTotalFreeSystemPtes
[MaximumPtePoolTypes
];
606 extern PFN_NUMBER MmTotalSystemDriverPages
;
607 extern ULONG MmCritsectTimeoutSeconds
;
608 extern PVOID MiSessionImageStart
;
609 extern PVOID MiSessionImageEnd
;
610 extern PMMPTE MiHighestUserPte
;
611 extern PMMPDE MiHighestUserPde
;
612 extern PFN_NUMBER MmSystemPageDirectory
[PD_COUNT
];
613 extern PMMPTE MmSharedUserDataPte
;
614 extern LIST_ENTRY MmProcessList
;
615 extern BOOLEAN MmZeroingPageThreadActive
;
616 extern KEVENT MmZeroingPageEvent
;
617 extern ULONG MmSystemPageColor
;
618 extern ULONG MmProcessColorSeed
;
619 extern PMMWSL MmWorkingSetList
;
620 extern PFN_NUMBER MiNumberOfFreePages
;
621 extern SIZE_T MmSessionViewSize
;
622 extern SIZE_T MmSessionPoolSize
;
623 extern SIZE_T MmSessionImageSize
;
624 extern PVOID MiSystemViewStart
;
625 extern PVOID MiSessionPoolEnd
; // 0xBE000000
626 extern PVOID MiSessionPoolStart
; // 0xBD000000
627 extern PVOID MiSessionViewStart
; // 0xBE000000
628 extern PVOID MiSessionSpaceWs
;
629 extern ULONG MmMaximumDeadKernelStacks
;
630 extern SLIST_HEADER MmDeadStackSListHead
;
631 extern MM_AVL_TABLE MmSectionBasedRoot
;
632 extern KGUARDED_MUTEX MmSectionBasedMutex
;
633 extern PVOID MmHighSectionBase
;
634 extern SIZE_T MmSystemLockPagesCount
;
635 extern ULONG_PTR MmSubsectionBase
;
636 extern LARGE_INTEGER MmCriticalSectionTimeout
;
637 extern LIST_ENTRY MmWorkingSetExpansionHead
;
638 extern KSPIN_LOCK MmExpansionLock
;
639 extern PETHREAD MiExpansionLockOwner
;
643 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType
)
645 return ((MemoryType
== LoaderFree
) ||
646 (MemoryType
== LoaderLoadedProgram
) ||
647 (MemoryType
== LoaderFirmwareTemporary
) ||
648 (MemoryType
== LoaderOsloaderStack
));
653 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType
)
655 return ((MemoryType
== LoaderFirmwarePermanent
) ||
656 (MemoryType
== LoaderSpecialMemory
) ||
657 (MemoryType
== LoaderHALCachedMemory
) ||
658 (MemoryType
== LoaderBBTMemory
));
664 MiIsUserPxe(PVOID Address
)
666 return ((ULONG_PTR
)Address
>> 7) == 0x1FFFFEDF6FB7DA0ULL
;
671 MiIsUserPpe(PVOID Address
)
673 return ((ULONG_PTR
)Address
>> 16) == 0xFFFFF6FB7DA0ULL
;
678 MiIsUserPde(PVOID Address
)
680 return ((ULONG_PTR
)Address
>> 25) == 0x7FFFFB7DA0ULL
;
685 MiIsUserPte(PVOID Address
)
687 return ((ULONG_PTR
)Address
>> 34) == 0x3FFFFDA0ULL
;
692 MiIsUserPde(PVOID Address
)
694 return ((Address
>= (PVOID
)MiAddressToPde(NULL
)) &&
695 (Address
<= (PVOID
)MiHighestUserPde
));
700 MiIsUserPte(PVOID Address
)
702 return (Address
<= (PVOID
)MiHighestUserPte
);
707 // Figures out the hardware bits for a PTE
711 MiDetermineUserGlobalPteMask(IN PVOID PointerPte
)
718 /* Make it valid and accessed */
719 TempPte
.u
.Hard
.Valid
= TRUE
;
720 MI_MAKE_ACCESSED_PAGE(&TempPte
);
722 /* Is this for user-mode? */
724 #if (_MI_PAGING_LEVELS == 4)
725 MiIsUserPxe(PointerPte
) ||
727 #if (_MI_PAGING_LEVELS >= 3)
728 MiIsUserPpe(PointerPte
) ||
730 MiIsUserPde(PointerPte
) ||
731 MiIsUserPte(PointerPte
))
733 /* Set the owner bit */
734 MI_MAKE_OWNER_PAGE(&TempPte
);
737 /* FIXME: We should also set the global bit */
739 /* Return the protection */
740 return TempPte
.u
.Long
;
744 // Creates a valid kernel PTE with the given protection
748 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte
,
749 IN PMMPTE MappingPte
,
750 IN ULONG_PTR ProtectionMask
,
751 IN PFN_NUMBER PageFrameNumber
)
753 /* Only valid for kernel, non-session PTEs */
754 ASSERT(MappingPte
> MiHighestUserPte
);
755 ASSERT(!MI_IS_SESSION_PTE(MappingPte
));
756 ASSERT((MappingPte
< (PMMPTE
)PDE_BASE
) || (MappingPte
> (PMMPTE
)PDE_TOP
));
759 *NewPte
= ValidKernelPte
;
761 /* Set the protection and page */
762 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
763 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
767 // Creates a valid PTE with the given protection
771 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte
,
772 IN PMMPTE MappingPte
,
773 IN ULONG_PTR ProtectionMask
,
774 IN PFN_NUMBER PageFrameNumber
)
776 /* Set the protection and page */
777 NewPte
->u
.Long
= MiDetermineUserGlobalPteMask(MappingPte
);
778 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
779 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
783 // Creates a valid user PTE with the given protection
787 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte
,
788 IN PMMPTE MappingPte
,
789 IN ULONG_PTR ProtectionMask
,
790 IN PFN_NUMBER PageFrameNumber
)
792 /* Only valid for kernel, non-session PTEs */
793 ASSERT(MappingPte
<= MiHighestUserPte
);
798 /* Set the protection and page */
799 NewPte
->u
.Hard
.Valid
= TRUE
;
800 NewPte
->u
.Hard
.Owner
= TRUE
;
801 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
802 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
807 // Builds a Prototype PTE for the address of the PTE
811 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte
,
812 IN PMMPTE PointerPte
)
816 /* Mark this as a prototype */
818 NewPte
->u
.Proto
.Prototype
= 1;
821 * Prototype PTEs are only valid in paged pool by design, this little trick
822 * lets us only use 30 bits for the adress of the PTE, as long as the area
823 * stays 1024MB At most.
825 Offset
= (ULONG_PTR
)PointerPte
- (ULONG_PTR
)MmPagedPoolStart
;
828 * 7 bits go in the "low" (but we assume the bottom 2 are zero)
829 * and the other 21 bits go in the "high"
831 NewPte
->u
.Proto
.ProtoAddressLow
= (Offset
& 0x1FC) >> 2;
832 NewPte
->u
.Proto
.ProtoAddressHigh
= (Offset
& 0x3FFFFE00) >> 9;
836 // Builds a Subsection PTE for the address of the Segment
840 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte
,
845 /* Mark this as a prototype */
847 NewPte
->u
.Subsect
.Prototype
= 1;
850 * Segments are only valid either in nonpaged pool. We store the 20 bit
851 * difference either from the top or bottom of nonpaged pool, giving a
852 * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
855 if ((ULONG_PTR
)Segment
< ((ULONG_PTR
)MmSubsectionBase
+ (128 * _1MB
)))
857 Offset
= (ULONG_PTR
)Segment
- (ULONG_PTR
)MmSubsectionBase
;
858 NewPte
->u
.Subsect
.WhichPool
= PagedPool
;
862 Offset
= (ULONG_PTR
)MmNonPagedPoolEnd
- (ULONG_PTR
)Segment
;
863 NewPte
->u
.Subsect
.WhichPool
= NonPagedPool
;
867 * 4 bits go in the "low" (but we assume the bottom 3 are zero)
868 * and the other 20 bits go in the "high"
870 NewPte
->u
.Subsect
.SubsectionAddressLow
= (Offset
& 0x78) >> 3;
871 NewPte
->u
.Subsect
.SubsectionAddressHigh
= (Offset
& 0xFFFFF80) >> 7;
876 MI_IS_MAPPED_PTE(PMMPTE PointerPte
)
878 /// \todo Make this reasonable code, this is UGLY!
879 return ((PointerPte
->u
.Long
& 0xFFFFFC01) != 0);
886 MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte
,
887 _In_ PFN_NUMBER Page
,
888 _In_ ULONG Protection
)
891 NewPte
->u
.Trans
.Transition
= 1;
892 NewPte
->u
.Trans
.Protection
= Protection
;
893 NewPte
->u
.Trans
.PageFrameNumber
= Page
;
897 // Returns if the page is physically resident (ie: a large page)
898 // FIXFIX: CISC/x86 only?
902 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address
)
906 /* Large pages are never paged out, always physically resident */
907 PointerPde
= MiAddressToPde(Address
);
908 return ((PointerPde
->u
.Hard
.LargePage
) && (PointerPde
->u
.Hard
.Valid
));
912 // Writes a valid PTE
916 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte
,
919 /* Write the valid PTE */
920 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
921 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
922 *PointerPte
= TempPte
;
926 // Updates a valid PTE
930 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte
,
933 /* Write the valid PTE */
934 ASSERT(PointerPte
->u
.Hard
.Valid
== 1);
935 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
936 ASSERT(PointerPte
->u
.Hard
.PageFrameNumber
== TempPte
.u
.Hard
.PageFrameNumber
);
937 *PointerPte
= TempPte
;
941 // Writes an invalid PTE
945 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte
,
948 /* Write the invalid PTE */
949 ASSERT(InvalidPte
.u
.Hard
.Valid
== 0);
950 ASSERT(InvalidPte
.u
.Long
!= 0);
951 *PointerPte
= InvalidPte
;
955 // Erase the PTE completely
959 MI_ERASE_PTE(IN PMMPTE PointerPte
)
961 /* Zero out the PTE */
962 ASSERT(PointerPte
->u
.Long
!= 0);
963 PointerPte
->u
.Long
= 0;
967 // Writes a valid PDE
971 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde
,
974 /* Write the valid PDE */
975 ASSERT(PointerPde
->u
.Hard
.Valid
== 0);
976 ASSERT(TempPde
.u
.Hard
.Valid
== 1);
977 *PointerPde
= TempPde
;
981 // Writes an invalid PDE
985 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde
,
988 /* Write the invalid PDE */
989 ASSERT(InvalidPde
.u
.Hard
.Valid
== 0);
990 ASSERT(InvalidPde
.u
.Long
!= 0);
991 *PointerPde
= InvalidPde
;
995 // Checks if the thread already owns a working set
999 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread
)
1001 /* If any of these are held, return TRUE */
1002 return ((Thread
->OwnsProcessWorkingSetExclusive
) ||
1003 (Thread
->OwnsProcessWorkingSetShared
) ||
1004 (Thread
->OwnsSystemWorkingSetExclusive
) ||
1005 (Thread
->OwnsSystemWorkingSetShared
) ||
1006 (Thread
->OwnsSessionWorkingSetExclusive
) ||
1007 (Thread
->OwnsSessionWorkingSetShared
));
1011 // Checks if the process owns the working set lock
1015 MI_WS_OWNER(IN PEPROCESS Process
)
1017 /* Check if this process is the owner, and that the thread owns the WS */
1018 if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive
== 0)
1020 DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1022 if (KeGetCurrentThread()->ApcState
.Process
!= &Process
->Pcb
)
1024 DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process
);
1026 return ((KeGetCurrentThread()->ApcState
.Process
== &Process
->Pcb
) &&
1027 ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive
) ||
1028 (PsGetCurrentThread()->OwnsProcessWorkingSetShared
)));
1032 // New ARM3<->RosMM PAGE Architecture
1036 MiIsRosSectionObject(IN PVOID Section
)
1038 PROS_SECTION_OBJECT RosSection
= Section
;
1039 if ((RosSection
->Type
== 'SC') && (RosSection
->Size
== 'TN')) return TRUE
;
1043 #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE)
1047 MiDecrementReferenceCount(
1049 IN PFN_NUMBER PageFrameIndex
1054 MI_IS_WS_UNSAFE(IN PEPROCESS Process
)
1056 return (Process
->Vm
.Flags
.AcquiredUnsafe
== TRUE
);
1060 // Locks the working set for the given process
1064 MiLockProcessWorkingSet(IN PEPROCESS Process
,
1067 /* Shouldn't already be owning the process working set */
1068 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1069 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1071 /* Block APCs, make sure that still nothing is already held */
1072 KeEnterGuardedRegion();
1073 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1075 /* Lock the working set */
1076 ExAcquirePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1078 /* Now claim that we own the lock */
1079 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1080 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1081 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1086 MiLockProcessWorkingSetShared(IN PEPROCESS Process
,
1089 /* Shouldn't already be owning the process working set */
1090 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1091 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1093 /* Block APCs, make sure that still nothing is already held */
1094 KeEnterGuardedRegion();
1095 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1097 /* Lock the working set */
1098 ExAcquirePushLockShared(&Process
->Vm
.WorkingSetMutex
);
1100 /* Now claim that we own the lock */
1101 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1102 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1103 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1104 Thread
->OwnsProcessWorkingSetShared
= TRUE
;
1109 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process
,
1112 /* Shouldn't already be owning the process working set */
1113 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1115 /* APCs must be blocked, make sure that still nothing is already held */
1116 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1117 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1119 /* Lock the working set */
1120 ExAcquirePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1122 /* Now claim that we own the lock */
1123 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1124 Process
->Vm
.Flags
.AcquiredUnsafe
= 1;
1125 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1126 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1130 // Unlocks the working set for the given process
1134 MiUnlockProcessWorkingSet(IN PEPROCESS Process
,
1137 /* Make sure we are the owner of a safe acquisition */
1138 ASSERT(MI_WS_OWNER(Process
));
1139 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1141 /* The thread doesn't own it anymore */
1142 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== TRUE
);
1143 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1145 /* Release the lock and re-enable APCs */
1146 ExReleasePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1147 KeLeaveGuardedRegion();
1151 // Unlocks the working set for the given process
1155 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process
,
1158 /* Make sure we are the owner of a safe acquisition (because shared) */
1159 ASSERT(MI_WS_OWNER(Process
));
1160 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1162 /* Ensure we are in a shared acquisition */
1163 ASSERT(Thread
->OwnsProcessWorkingSetShared
== TRUE
);
1164 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1166 /* Don't claim the lock anylonger */
1167 Thread
->OwnsProcessWorkingSetShared
= FALSE
;
1169 /* Release the lock and re-enable APCs */
1170 ExReleasePushLockShared(&Process
->Vm
.WorkingSetMutex
);
1171 KeLeaveGuardedRegion();
1175 // Unlocks the working set for the given process
1179 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process
,
1182 /* Make sure we are the owner of an unsafe acquisition */
1183 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1184 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1185 ASSERT(MI_WS_OWNER(Process
));
1186 ASSERT(MI_IS_WS_UNSAFE(Process
));
1188 /* No longer unsafe */
1189 Process
->Vm
.Flags
.AcquiredUnsafe
= 0;
1191 /* The thread doesn't own it anymore */
1192 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== TRUE
);
1193 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1195 /* Release the lock but don't touch APC state */
1196 ExReleasePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1197 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1201 // Locks the working set
1205 MiLockWorkingSet(IN PETHREAD Thread
,
1206 IN PMMSUPPORT WorkingSet
)
1209 KeEnterGuardedRegion();
1211 /* Working set should be in global memory */
1212 ASSERT(MI_IS_SESSION_ADDRESS((PVOID
)WorkingSet
) == FALSE
);
1214 /* Thread shouldn't already be owning something */
1215 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1217 /* Lock this working set */
1218 ExAcquirePushLockExclusive(&WorkingSet
->WorkingSetMutex
);
1220 /* Which working set is this? */
1221 if (WorkingSet
== &MmSystemCacheWs
)
1223 /* Own the system working set */
1224 ASSERT((Thread
->OwnsSystemWorkingSetExclusive
== FALSE
) &&
1225 (Thread
->OwnsSystemWorkingSetShared
== FALSE
));
1226 Thread
->OwnsSystemWorkingSetExclusive
= TRUE
;
1228 else if (WorkingSet
->Flags
.SessionSpace
)
1230 /* Own the session working set */
1231 ASSERT((Thread
->OwnsSessionWorkingSetExclusive
== FALSE
) &&
1232 (Thread
->OwnsSessionWorkingSetShared
== FALSE
));
1233 Thread
->OwnsSessionWorkingSetExclusive
= TRUE
;
1237 /* Own the process working set */
1238 ASSERT((Thread
->OwnsProcessWorkingSetExclusive
== FALSE
) &&
1239 (Thread
->OwnsProcessWorkingSetShared
== FALSE
));
1240 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1245 // Unlocks the working set
1249 MiUnlockWorkingSet(IN PETHREAD Thread
,
1250 IN PMMSUPPORT WorkingSet
)
1252 /* Working set should be in global memory */
1253 ASSERT(MI_IS_SESSION_ADDRESS((PVOID
)WorkingSet
) == FALSE
);
1255 /* Which working set is this? */
1256 if (WorkingSet
== &MmSystemCacheWs
)
1258 /* Release the system working set */
1259 ASSERT((Thread
->OwnsSystemWorkingSetExclusive
== TRUE
) ||
1260 (Thread
->OwnsSystemWorkingSetShared
== TRUE
));
1261 Thread
->OwnsSystemWorkingSetExclusive
= FALSE
;
1263 else if (WorkingSet
->Flags
.SessionSpace
)
1265 /* Release the session working set */
1266 ASSERT((Thread
->OwnsSessionWorkingSetExclusive
== TRUE
) ||
1267 (Thread
->OwnsSessionWorkingSetShared
== TRUE
));
1268 Thread
->OwnsSessionWorkingSetExclusive
= 0;
1272 /* Release the process working set */
1273 ASSERT((Thread
->OwnsProcessWorkingSetExclusive
) ||
1274 (Thread
->OwnsProcessWorkingSetShared
));
1275 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1278 /* Release the working set lock */
1279 ExReleasePushLockExclusive(&WorkingSet
->WorkingSetMutex
);
1282 KeLeaveGuardedRegion();
1287 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process
,
1290 OUT PBOOLEAN Shared
)
1292 ASSERT(MI_WS_OWNER(Process
));
1294 /* Check if the current owner is unsafe */
1295 if (MI_IS_WS_UNSAFE(Process
))
1297 /* Release unsafely */
1298 MiUnlockProcessWorkingSetUnsafe(Process
, Thread
);
1302 else if (Thread
->OwnsProcessWorkingSetExclusive
== 1)
1304 /* Owner is safe and exclusive, release normally */
1305 MiUnlockProcessWorkingSet(Process
, Thread
);
1311 /* Owner is shared (implies safe), release normally */
1312 MiUnlockProcessWorkingSetShared(Process
, Thread
);
1320 MiLockProcessWorkingSetForFault(IN PEPROCESS Process
,
1325 /* Check if this was a safe lock or not */
1330 /* Reacquire safely & shared */
1331 MiLockProcessWorkingSetShared(Process
, Thread
);
1335 /* Reacquire safely */
1336 MiLockProcessWorkingSet(Process
, Thread
);
1341 /* Unsafe lock cannot be shared */
1342 ASSERT(Shared
== FALSE
);
1343 /* Reacquire unsafely */
1344 MiLockProcessWorkingSetUnsafe(Process
, Thread
);
1350 MiAcquireExpansionLock(VOID
)
1354 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1355 KeAcquireSpinLock(&MmExpansionLock
, &OldIrql
);
1356 ASSERT(MiExpansionLockOwner
== NULL
);
1357 MiExpansionLockOwner
= PsGetCurrentThread();
1363 MiReleaseExpansionLock(KIRQL OldIrql
)
1365 ASSERT(MiExpansionLockOwner
== PsGetCurrentThread());
1366 MiExpansionLockOwner
= NULL
;
1367 KeReleaseSpinLock(&MmExpansionLock
, OldIrql
);
1368 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1372 // Returns the ProtoPTE inside a VAD for the given VPN
1376 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad
,
1381 /* Find the offset within the VAD's prototype PTEs */
1382 ProtoPte
= Vad
->FirstPrototypePte
+ (Vpn
- Vad
->StartingVpn
);
1383 ASSERT(ProtoPte
<= Vad
->LastContiguousPte
);
1388 // Returns the PFN Database entry for the given page number
1389 // Warning: This is not necessarily a valid PFN database entry!
1393 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn
)
1396 return &MmPfnDatabase
[Pfn
];
1400 // Drops a locked page without dereferencing it
1404 MiDropLockCount(IN PMMPFN Pfn1
)
1406 /* This page shouldn't be locked, but it should be valid */
1407 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
1408 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1410 /* Is this the last reference to the page */
1411 if (Pfn1
->u3
.e2
.ReferenceCount
== 1)
1413 /* It better not be valid */
1414 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1416 /* Is it a prototype PTE? */
1417 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1418 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1420 /* FIXME: We should return commit */
1421 DPRINT1("Not returning commit for prototype PTE\n");
1424 /* Update the counter */
1425 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1430 // Drops a locked page and dereferences it
1434 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1
)
1436 USHORT RefCount
, OldRefCount
;
1437 PFN_NUMBER PageFrameIndex
;
1439 /* Loop while we decrement the page successfully */
1442 /* There should be at least one reference */
1443 OldRefCount
= Pfn1
->u3
.e2
.ReferenceCount
;
1444 ASSERT(OldRefCount
!= 0);
1446 /* Are we the last one */
1447 if (OldRefCount
== 1)
1449 /* The page shoudln't be shared not active at this point */
1450 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 1);
1451 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1452 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1454 /* Is it a prototype PTE? */
1455 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1456 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1458 /* FIXME: We should return commit */
1459 DPRINT1("Not returning commit for prototype PTE\n");
1462 /* Update the counter, and drop a reference the long way */
1463 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1464 PageFrameIndex
= MiGetPfnEntryIndex(Pfn1
);
1465 MiDecrementReferenceCount(Pfn1
, PageFrameIndex
);
1469 /* Drop a reference the short way, and that's it */
1470 RefCount
= InterlockedCompareExchange16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
,
1473 ASSERT(RefCount
!= 0);
1474 } while (OldRefCount
!= RefCount
);
1476 /* If we got here, there should be more than one reference */
1477 ASSERT(RefCount
> 1);
1480 /* Is it still being shared? */
1481 if (Pfn1
->u2
.ShareCount
>= 1)
1483 /* Then it should be valid */
1484 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1486 /* Is it a prototype PTE? */
1487 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1488 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1490 /* We don't handle ethis */
1494 /* Update the counter */
1495 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1501 // References a locked page and updates the counter
1502 // Used in MmProbeAndLockPages to handle different edge cases
1506 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1508 USHORT RefCount
, OldRefCount
;
1511 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
1513 /* Does ARM3 own the page? */
1514 if (MI_IS_ROS_PFN(Pfn1
))
1516 /* ReactOS Mm doesn't track share count */
1517 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1521 /* On ARM3 pages, we should see a valid share count */
1522 ASSERT((Pfn1
->u2
.ShareCount
!= 0) && (Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
));
1524 /* Is it a prototype PTE? */
1525 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1526 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1528 /* FIXME: We should charge commit */
1529 DPRINT1("Not charging commit for prototype PTE\n");
1533 /* More locked pages! */
1534 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1536 /* Loop trying to update the reference count */
1539 /* Get the current reference count, make sure it's valid */
1540 OldRefCount
= Pfn1
->u3
.e2
.ReferenceCount
;
1541 ASSERT(OldRefCount
!= 0);
1542 ASSERT(OldRefCount
< 2500);
1544 /* Bump it up by one */
1545 RefCount
= InterlockedCompareExchange16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
,
1548 ASSERT(RefCount
!= 0);
1549 } while (OldRefCount
!= RefCount
);
1551 /* Was this the first lock attempt? If not, undo our bump */
1552 if (OldRefCount
!= 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1556 // References a locked page and updates the counter
1557 // Used in all other cases except MmProbeAndLockPages
1561 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1565 /* Is it a prototype PTE? */
1566 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1567 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1569 /* FIXME: We should charge commit */
1570 DPRINT1("Not charging commit for prototype PTE\n");
1573 /* More locked pages! */
1574 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1576 /* Update the reference count */
1577 NewRefCount
= InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1578 if (NewRefCount
== 2)
1580 /* Is it locked or shared? */
1581 if (Pfn1
->u2
.ShareCount
)
1583 /* It's shared, so make sure it's active */
1584 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1588 /* It's locked, so we shouldn't lock again */
1589 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1594 /* Someone had already locked the page, so undo our bump */
1595 ASSERT(NewRefCount
< 2500);
1596 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1601 // References a locked page and updates the counter
1602 // Used in all other cases except MmProbeAndLockPages
1606 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1610 /* Make sure the page isn't used yet */
1611 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1612 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1614 /* Is it a prototype PTE? */
1615 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1616 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1618 /* FIXME: We should charge commit */
1619 DPRINT1("Not charging commit for prototype PTE\n");
1622 /* More locked pages! */
1623 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1625 /* Update the reference count */
1626 NewRefCount
= InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1627 if (NewRefCount
!= 1)
1629 /* Someone had already locked the page, so undo our bump */
1630 ASSERT(NewRefCount
< 2500);
1631 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1637 MiIncrementPageTableReferences(IN PVOID Address
)
1641 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1644 ASSERT(*RefCount
<= PTE_PER_PAGE
);
1649 MiDecrementPageTableReferences(IN PVOID Address
)
1653 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1656 ASSERT(*RefCount
< PTE_PER_PAGE
);
1661 MiQueryPageTableReferences(IN PVOID Address
)
1665 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1674 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1679 MiInitializeSessionSpaceLayout(VOID
);
1683 MiInitMachineDependent(
1684 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1689 MiComputeColorInformation(
1696 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1701 MiInitializeColorTables(
1707 MiInitializePfnDatabase(
1708 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1713 MiInitializeSessionWsSupport(
1719 MiInitializeSessionIds(
1725 MiInitializeMemoryEvents(
1732 IN PFN_NUMBER PageCount
1735 PPHYSICAL_MEMORY_DESCRIPTOR
1737 MmInitializeMemoryLimits(
1738 IN PLOADER_PARAMETER_BLOCK LoaderBlock
,
1739 IN PBOOLEAN IncludeType
1744 MiPagesInLoaderBlock(
1745 IN PLOADER_PARAMETER_BLOCK LoaderBlock
,
1746 IN PBOOLEAN IncludeType
1752 IN PVOID AddressStart
,
1758 MiRosProtectVirtualMemory(
1759 IN PEPROCESS Process
,
1760 IN OUT PVOID
*BaseAddress
,
1761 IN OUT PSIZE_T NumberOfBytesToProtect
,
1762 IN ULONG NewAccessProtection
,
1763 OUT PULONG OldAccessProtection OPTIONAL
1769 IN BOOLEAN StoreInstruction
,
1771 IN KPROCESSOR_MODE Mode
,
1772 IN PVOID TrapInformation
1777 MiCheckPdeForPagedPool(
1783 MiInitializeNonPagedPool(
1789 MiInitializeNonPagedPoolThresholds(
1795 MiInitializePoolEvents(
1802 IN POOL_TYPE PoolType
,// FIXFIX: This should go in ex.h after the pool merge
1803 IN ULONG Threshold
//
1806 // FIXFIX: THIS ONE TOO
1810 ExInitializePoolDescriptor(
1811 IN PPOOL_DESCRIPTOR PoolDescriptor
,
1812 IN POOL_TYPE PoolType
,
1820 MiInitializeSessionPool(
1826 MiInitializeSystemPtes(
1827 IN PMMPTE StartingPte
,
1828 IN ULONG NumberOfPtes
,
1829 IN MMSYSTEM_PTE_POOL_TYPE PoolType
1834 MiReserveSystemPtes(
1835 IN ULONG NumberOfPtes
,
1836 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1841 MiReleaseSystemPtes(
1842 IN PMMPTE StartingPte
,
1843 IN ULONG NumberOfPtes
,
1844 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1850 MiFindContiguousPages(
1851 IN PFN_NUMBER LowestPfn
,
1852 IN PFN_NUMBER HighestPfn
,
1853 IN PFN_NUMBER BoundaryPfn
,
1854 IN PFN_NUMBER SizeInPages
,
1855 IN MEMORY_CACHING_TYPE CacheType
1860 MiCheckForContiguousMemory(
1861 IN PVOID BaseAddress
,
1862 IN PFN_NUMBER BaseAddressPages
,
1863 IN PFN_NUMBER SizeInPages
,
1864 IN PFN_NUMBER LowestPfn
,
1865 IN PFN_NUMBER HighestPfn
,
1866 IN PFN_NUMBER BoundaryPfn
,
1867 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1872 MiAllocatePagesForMdl(
1873 IN PHYSICAL_ADDRESS LowAddress
,
1874 IN PHYSICAL_ADDRESS HighAddress
,
1875 IN PHYSICAL_ADDRESS SkipBytes
,
1876 IN SIZE_T TotalBytes
,
1877 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
,
1884 IN PMMPFNLIST ListHead
,
1885 IN PFN_NUMBER PageFrameIndex
1890 MiUnlinkFreeOrZeroedPage(
1896 MiUnlinkPageFromList(
1903 IN PFN_NUMBER PageFrameIndex
,
1904 IN PMMPTE PointerPte
,
1910 MiInitializeAndChargePfn(
1911 OUT PPFN_NUMBER PageFrameIndex
,
1912 IN PMMPDE PointerPde
,
1913 IN PFN_NUMBER ContainingPageFrame
,
1914 IN BOOLEAN SessionAllocation
1919 MiInitializePfnAndMakePteValid(
1920 IN PFN_NUMBER PageFrameIndex
,
1921 IN PMMPTE PointerPte
,
1927 MiInitializePfnForOtherProcess(
1928 IN PFN_NUMBER PageFrameIndex
,
1929 IN PVOID PteAddress
,
1930 IN PFN_NUMBER PteFrame
1935 MiDecrementShareCount(
1937 IN PFN_NUMBER PageFrameIndex
1955 IN PFN_NUMBER PageFrameIndex
1960 MiInsertPageInFreeList(
1961 IN PFN_NUMBER PageFrameIndex
1966 MiDeleteSystemPageableVm(
1967 IN PMMPTE PointerPte
,
1968 IN PFN_NUMBER PageCount
,
1970 OUT PPFN_NUMBER ValidPages
1975 MiGetPageProtection(
1976 IN PMMPTE PointerPte
1979 PLDR_DATA_TABLE_ENTRY
1981 MiLookupDataTableEntry(
1987 MiInitializeDriverLargePageList(
1993 MiInitializeLargePageSupport(
2012 IN PVOID VirtualAddress
2017 MiCheckForConflictingNode(
2018 IN ULONG_PTR StartVpn
,
2019 IN ULONG_PTR EndVpn
,
2020 IN PMM_AVL_TABLE Table
,
2021 OUT PMMADDRESS_NODE
*NodeOrParent
2026 MiFindEmptyAddressRangeDownTree(
2028 IN ULONG_PTR BoundaryAddress
,
2029 IN ULONG_PTR Alignment
,
2030 IN PMM_AVL_TABLE Table
,
2031 OUT PULONG_PTR Base
,
2032 OUT PMMADDRESS_NODE
*Parent
2037 MiFindEmptyAddressRangeDownBasedTree(
2039 IN ULONG_PTR BoundaryAddress
,
2040 IN ULONG_PTR Alignment
,
2041 IN PMM_AVL_TABLE Table
,
2047 MiFindEmptyAddressRangeInTree(
2049 IN ULONG_PTR Alignment
,
2050 IN PMM_AVL_TABLE Table
,
2051 OUT PMMADDRESS_NODE
*PreviousVad
,
2061 IN ULONG ProtectionMask
2068 _Inout_ PMM_AVL_TABLE VadRoot
);
2074 _In_ ULONG_PTR
*BaseAddress
,
2075 _In_ SIZE_T ViewSize
,
2076 _In_ ULONG_PTR HighestAddress
,
2077 _In_ ULONG_PTR Alignment
,
2078 _In_ ULONG AllocationType
);
2082 MiInsertBasedSection(
2088 MiUnmapViewOfSection(
2089 IN PEPROCESS Process
,
2090 IN PVOID BaseAddress
,
2096 MiRosUnmapViewOfSection(
2097 IN PEPROCESS Process
,
2098 IN PVOID BaseAddress
,
2099 IN BOOLEAN SkipDebuggerNotify
2105 IN PMM_AVL_TABLE Table
,
2106 IN PMMADDRESS_NODE NewNode
,
2107 PMMADDRESS_NODE Parent
,
2108 TABLE_SEARCH_RESULT Result
2114 IN PMMADDRESS_NODE Node
,
2115 IN PMM_AVL_TABLE Table
2121 IN PMMADDRESS_NODE Node
2127 IN PMMADDRESS_NODE Node
2132 MiInitializeSystemSpaceMap(
2133 IN PMMSESSION InputSession OPTIONAL
2138 MiSessionRemoveProcess(
2144 MiReleaseProcessReferenceToSessionDataPage(
2145 IN PMM_SESSION_SPACE SessionGlobal
2150 MiSessionAddProcess(
2151 IN PEPROCESS NewProcess
2156 MiSessionCommitPageTables(
2163 MiMakeProtectionMask(
2169 MiDeleteVirtualAddresses(
2171 IN ULONG_PTR EndingAddress
,
2178 IN PMMPTE PointerPte
,
2179 IN PVOID VirtualAddress
,
2180 IN PEPROCESS CurrentProcess
,
2181 IN PMMPTE PrototypePte
2186 MiMakeSystemAddressValid(
2187 IN PVOID PageTableVirtualAddress
,
2188 IN PEPROCESS CurrentProcess
2193 MiMakeSystemAddressValidPfn(
2194 IN PVOID VirtualAddress
,
2201 IN PEPROCESS CurrentProcess
,
2214 MiDeleteARM3Section(
2220 MiQueryMemorySectionName(
2221 IN HANDLE ProcessHandle
,
2222 IN PVOID BaseAddress
,
2223 OUT PVOID MemoryInformation
,
2224 IN SIZE_T MemoryInformationLength
,
2225 OUT PSIZE_T ReturnLength
2230 MiRosUnmapViewInSystemSpace(
2236 MmDeterminePoolType(
2237 IN PVOID PoolAddress
2242 MiMakePdeExistAndMakeValid(
2243 IN PMMPDE PointerPde
,
2244 IN PEPROCESS TargetProcess
,
2249 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2250 // free pages are available. In some scenarios, we don't/can't run that piece of
2251 // code and would rather only have a real zero page. If we can't have a zero page,
2252 // then we'd like to have our own code to grab a free page and zero it out, by
2253 // using MiRemoveAnyPage. This macro implements this.
2257 MiRemoveZeroPageSafe(IN ULONG Color
)
2259 if (MmFreePagesByColor
[ZeroedPageList
][Color
].Flink
!= LIST_HEAD
) return MiRemoveZeroPage(Color
);
2263 #if (_MI_PAGING_LEVELS == 2)
2266 MiSynchronizeSystemPde(PMMPDE PointerPde
)
2271 /* Get the Index from the PDE */
2272 Index
= ((ULONG_PTR
)PointerPde
& (SYSTEM_PD_SIZE
- 1)) / sizeof(MMPTE
);
2274 /* Copy the PDE from the double-mapped system page directory */
2275 SystemPde
= MmSystemPagePtes
[Index
];
2276 *PointerPde
= SystemPde
;
2278 /* Make sure we re-read the PDE and PTE */
2279 KeMemoryBarrierWithoutFence();
2281 /* Return, if we had success */
2282 return SystemPde
.u
.Hard
.Valid
!= 0;