2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/miarm.h
5 * PURPOSE: ARM Memory Manager Header
6 * PROGRAMMERS: ReactOS Portable Systems Group
11 #define MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING ((255 * _1MB) >> PAGE_SHIFT)
12 #define MI_MIN_PAGES_FOR_SYSPTE_TUNING ((19 * _1MB) >> PAGE_SHIFT)
13 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST ((32 * _1MB) >> PAGE_SHIFT)
14 #define MI_MIN_PAGES_FOR_SYSPTE_BOOST_BOOST ((256 * _1MB) >> PAGE_SHIFT)
15 #define MI_MAX_INIT_NONPAGED_POOL_SIZE (128 * _1MB)
16 #define MI_MAX_NONPAGED_POOL_SIZE (128 * _1MB)
17 #define MI_MAX_FREE_PAGE_LISTS 4
19 #define MI_MIN_INIT_PAGED_POOLSIZE (32 * _1MB)
21 #define MI_SESSION_VIEW_SIZE (48 * _1MB)
22 #define MI_SESSION_POOL_SIZE (16 * _1MB)
23 #define MI_SESSION_IMAGE_SIZE (8 * _1MB)
24 #define MI_SESSION_WORKING_SET_SIZE (4 * _1MB)
25 #define MI_SESSION_SIZE (MI_SESSION_VIEW_SIZE + \
26 MI_SESSION_POOL_SIZE + \
27 MI_SESSION_IMAGE_SIZE + \
28 MI_SESSION_WORKING_SET_SIZE)
30 #define MI_SYSTEM_VIEW_SIZE (32 * _1MB)
32 #define MI_HIGHEST_USER_ADDRESS (PVOID)0x7FFEFFFF
33 #define MI_USER_PROBE_ADDRESS (PVOID)0x7FFF0000
34 #define MI_DEFAULT_SYSTEM_RANGE_START (PVOID)0x80000000
35 #define MI_SYSTEM_CACHE_WS_START (PVOID)0xC0C00000
36 #define MI_PAGED_POOL_START (PVOID)0xE1000000
37 #define MI_NONPAGED_POOL_END (PVOID)0xFFBE0000
38 #define MI_DEBUG_MAPPING (PVOID)0xFFBFF000
40 #define MI_SYSTEM_PTE_BASE (PVOID)MiAddressToPte(NULL)
42 #define MI_MIN_SECONDARY_COLORS 8
43 #define MI_SECONDARY_COLORS 64
44 #define MI_MAX_SECONDARY_COLORS 1024
46 #define MI_MIN_ALLOCATION_FRAGMENT (4 * _1KB)
47 #define MI_ALLOCATION_FRAGMENT (64 * _1KB)
48 #define MI_MAX_ALLOCATION_FRAGMENT (2 * _1MB)
50 #define MM_HIGHEST_VAD_ADDRESS \
51 (PVOID)((ULONG_PTR)MM_HIGHEST_USER_ADDRESS - (16 * PAGE_SIZE))
52 #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS
54 #define MI_DEFAULT_SYSTEM_PTE_COUNT 50000
55 #define MI_MAX_ZERO_BITS 21
57 #endif /* !_M_AMD64 */
59 /* Make the code cleaner with some definitions for size multiples */
61 #define _1MB (1024 * _1KB)
62 #define _1GB (1024 * _1MB)
64 /* Everyone loves 64K */
65 #define _64K (64 * _1KB)
67 /* Area mapped by a PDE */
68 #define PDE_MAPPED_VA (PTE_COUNT * PAGE_SIZE)
70 /* Size of a page table */
71 #define PT_SIZE (PTE_COUNT * sizeof(MMPTE))
73 /* Size of a page directory */
74 #define PD_SIZE (PDE_COUNT * sizeof(MMPDE))
76 /* Size of all page directories for a process */
77 #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE)
79 /* Architecture specific count of PDEs in a directory, and count of PTEs in a PT */
82 #define PDE_COUNT 1024
83 #define PTE_COUNT 1024
84 C_ASSERT(SYSTEM_PD_SIZE
== PAGE_SIZE
);
85 #define MiIsPteOnPdeBoundary(PointerPte) \
86 ((((ULONG_PTR)PointerPte) & (PAGE_SIZE - 1)) == 0)
89 #define PDE_COUNT 4096
92 #define PD_COUNT PPE_PER_PAGE
93 #define PDE_COUNT PDE_PER_PAGE
94 #define PTE_COUNT PTE_PER_PAGE
98 // Protection Bits part of the internal memory manager Protection Mask, from:
99 // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel
100 // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants
101 // and public assertions.
103 #define MM_ZERO_ACCESS 0
104 #define MM_READONLY 1
106 #define MM_EXECUTE_READ 3
107 #define MM_READWRITE 4
108 #define MM_WRITECOPY 5
109 #define MM_EXECUTE_READWRITE 6
110 #define MM_EXECUTE_WRITECOPY 7
111 #define MM_PROTECT_ACCESS 7
114 // These are flags on top of the actual protection mask
116 #define MM_NOCACHE 0x08
117 #define MM_GUARDPAGE 0x10
118 #define MM_WRITECOMBINE 0x18
119 #define MM_PROTECT_SPECIAL 0x18
122 // These are special cases
124 #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE)
125 #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE)
126 #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE)
127 #define MM_INVALID_PROTECTION 0xFFFFFFFF
130 // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits
131 // The Memory Manager's definition define the attributes that must be preserved
132 // and these PTE definitions describe the attributes in the hardware sense. This
133 // helps deal with hardware differences between the actual boolean expression of
136 // For example, in the logical attributes, we want to express read-only as a flag
137 // but on x86, it is writability that must be set. On the other hand, on x86, just
138 // like in the kernel, it is disabling the caches that requires a special flag,
139 // while on certain architectures such as ARM, it is enabling the cache which
142 #if defined(_M_IX86) || defined(_M_AMD64)
146 #define PTE_READONLY 0 // Doesn't exist on x86
147 #define PTE_EXECUTE 0 // Not worrying about NX yet
148 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
149 #define PTE_READWRITE 0x2
150 #define PTE_WRITECOPY 0x200
151 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet
152 #define PTE_EXECUTE_WRITECOPY 0x200
153 #define PTE_PROTOTYPE 0x400
158 #define PTE_VALID 0x1
159 #define PTE_ACCESSED 0x20
160 #define PTE_DIRTY 0x40
165 #define PTE_ENABLE_CACHE 0
166 #define PTE_DISABLE_CACHE 0x10
167 #define PTE_WRITECOMBINED_CACHE 0x10
168 #elif defined(_M_ARM)
169 #define PTE_READONLY 0x200
170 #define PTE_EXECUTE 0 // Not worrying about NX yet
171 #define PTE_EXECUTE_READ 0 // Not worrying about NX yet
172 #define PTE_READWRITE 0 // Doesn't exist on ARM
173 #define PTE_WRITECOPY 0 // Doesn't exist on ARM
174 #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet
175 #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet
176 #define PTE_PROTOTYPE 0x400 // Using the Shared bit
180 #define PTE_ENABLE_CACHE 0
181 #define PTE_DISABLE_CACHE 0x10
182 #define PTE_WRITECOMBINED_CACHE 0x10
184 #error Define these please!
187 extern const ULONG_PTR MmProtectToPteMask
[32];
188 extern const ULONG MmProtectToValue
[32];
191 // Assertions for session images, addresses, and PTEs
193 #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \
194 (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd))
196 #define MI_IS_SESSION_ADDRESS(Address) \
197 (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd))
199 #define MI_IS_SESSION_PTE(Pte) \
200 ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte))
202 #define MI_IS_PAGE_TABLE_ADDRESS(Address) \
203 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP))
205 #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \
206 (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP))
208 #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \
209 (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd))
212 // Corresponds to MMPTE_SOFTWARE.Protection
215 #define MM_PTE_SOFTWARE_PROTECTION_BITS 5
217 #define MM_PTE_SOFTWARE_PROTECTION_BITS 6
219 #define MM_PTE_SOFTWARE_PROTECTION_BITS 5
221 #error Define these please!
225 // Creates a software PTE with the given protection
227 #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS))
230 // Marks a PTE as deleted
232 #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1))
233 #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1)
236 // Special values for LoadedImports
238 #define MM_SYSLDR_NO_IMPORTS (PVOID)0xFFFFFFFE
239 #define MM_SYSLDR_BOOT_LOADED (PVOID)0xFFFFFFFF
240 #define MM_SYSLDR_SINGLE_ENTRY 0x1
243 // Number of initial session IDs
245 #define MI_INITIAL_SESSION_IDS 64
247 #if defined(_M_IX86) || defined(_M_ARM)
251 #define LIST_HEAD 0xFFFFFFFF
254 // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits,
255 // we need a manual definition suited to the number of bits in the PteFrame.
256 // This is used as a LIST_HEAD for the colored list
258 #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF
259 #elif defined(_M_AMD64)
260 #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL
261 #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL
263 #error Define these please!
267 // Special IRQL value (found in assertions)
269 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
272 // Returns the color of a page
274 #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask)
275 #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor))
276 #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
280 // Decodes a Prototype PTE into the underlying PTE
282 #define MiProtoPteToPte(x) \
283 (PMMPTE)((ULONG_PTR)MmPagedPoolStart + \
284 (((x)->u.Proto.ProtoAddressHigh << 9) | (x)->u.Proto.ProtoAddressLow << 2))
287 // Decodes a Prototype PTE into the underlying PTE
289 #define MiSubsectionPteToSubsection(x) \
290 ((x)->u.Subsect.WhichPool == PagedPool) ? \
291 (PMMPTE)((ULONG_PTR)MmSubsectionBase + \
292 (((x)->u.Subsect.SubsectionAddressHigh << 7) | \
293 (x)->u.Subsect.SubsectionAddressLow << 3)) : \
294 (PMMPTE)((ULONG_PTR)MmNonPagedPoolEnd - \
295 (((x)->u.Subsect.SubsectionAddressHigh << 7) | \
296 (x)->u.Subsect.SubsectionAddressLow << 3))
300 // Prototype PTEs that don't yet have a pagefile association
303 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL
305 #define MI_PTE_LOOKUP_NEEDED 0xFFFFF
309 // Number of session lists in the MM_SESSIONS_SPACE structure
311 #if defined(_M_AMD64)
312 #define SESSION_POOL_LOOKASIDES 21
313 #elif defined(_M_IX86)
314 #define SESSION_POOL_LOOKASIDES 26
320 // Number of session data and tag pages
322 #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
323 #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE)
326 // Used by MiCheckSecuredVad
328 #define MM_READ_WRITE_ALLOWED 11
329 #define MM_READ_ONLY_ALLOWED 10
330 #define MM_NO_ACCESS_ALLOWED 01
331 #define MM_DELETE_CHECK 85
334 // System views are binned into 64K chunks
336 #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K
339 // FIXFIX: These should go in ex.h after the pool merge
342 #define POOL_BLOCK_SIZE 16
344 #define POOL_BLOCK_SIZE 8
346 #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE)
347 #define BASE_POOL_TYPE_MASK 1
348 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE))
351 // Pool debugging/analysis/tracing flags
353 #define POOL_FLAG_CHECK_TIMERS 0x1
354 #define POOL_FLAG_CHECK_WORKERS 0x2
355 #define POOL_FLAG_CHECK_RESOURCES 0x4
356 #define POOL_FLAG_VERIFIER 0x8
357 #define POOL_FLAG_CHECK_DEADLOCK 0x10
358 #define POOL_FLAG_SPECIAL_POOL 0x20
359 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40
360 #define POOL_FLAG_CRASH_ON_FAILURE 0x80
363 // BAD_POOL_HEADER codes during pool bugcheck
365 #define POOL_CORRUPTED_LIST 3
366 #define POOL_SIZE_OR_INDEX_MISMATCH 5
367 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6
368 #define POOL_HEADER_NOT_ALIGNED 7
369 #define POOL_HEADER_IS_ZERO 8
370 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9
371 #define POOL_ENTRY_NOT_FOUND 10
374 // BAD_POOL_CALLER codes during pool bugcheck
376 #define POOL_ENTRY_CORRUPTED 1
377 #define POOL_ENTRY_ALREADY_FREE 6
378 #define POOL_ENTRY_NOT_ALLOCATED 7
379 #define POOL_ALLOC_IRQL_INVALID 8
380 #define POOL_FREE_IRQL_INVALID 9
381 #define POOL_BILLED_PROCESS_INVALID 13
382 #define POOL_HEADER_SIZE_INVALID 32
384 typedef struct _POOL_DESCRIPTOR
389 ULONG RunningDeAllocs
;
395 LONG PendingFreeDepth
;
398 LIST_ENTRY ListHeads
[POOL_LISTS_PER_PAGE
];
399 } POOL_DESCRIPTOR
, *PPOOL_DESCRIPTOR
;
401 typedef struct _POOL_HEADER
408 USHORT PreviousSize
:8;
413 USHORT PreviousSize
:9;
427 PEPROCESS ProcessBilled
;
433 USHORT AllocatorBackTraceIndex
;
437 } POOL_HEADER
, *PPOOL_HEADER
;
439 C_ASSERT(sizeof(POOL_HEADER
) == POOL_BLOCK_SIZE
);
440 C_ASSERT(POOL_BLOCK_SIZE
== sizeof(LIST_ENTRY
));
442 typedef struct _POOL_TRACKER_TABLE
447 SIZE_T NonPagedBytes
;
451 } POOL_TRACKER_TABLE
, *PPOOL_TRACKER_TABLE
;
453 typedef struct _POOL_TRACKER_BIG_PAGES
459 } POOL_TRACKER_BIG_PAGES
, *PPOOL_TRACKER_BIG_PAGES
;
461 extern ULONG ExpNumberOfPagedPools
;
462 extern POOL_DESCRIPTOR NonPagedPoolDescriptor
;
463 extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor
[16 + 1];
464 extern PPOOL_TRACKER_TABLE PoolTrackTable
;
470 typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY
473 UNICODE_STRING BaseName
;
474 } MI_LARGE_PAGE_DRIVER_ENTRY
, *PMI_LARGE_PAGE_DRIVER_ENTRY
;
476 typedef enum _MMSYSTEM_PTE_POOL_TYPE
479 NonPagedPoolExpansion
,
481 } MMSYSTEM_PTE_POOL_TYPE
;
483 typedef enum _MI_PFN_CACHE_ATTRIBUTE
489 } MI_PFN_CACHE_ATTRIBUTE
, *PMI_PFN_CACHE_ATTRIBUTE
;
491 typedef struct _PHYSICAL_MEMORY_RUN
494 PFN_NUMBER PageCount
;
495 } PHYSICAL_MEMORY_RUN
, *PPHYSICAL_MEMORY_RUN
;
497 typedef struct _PHYSICAL_MEMORY_DESCRIPTOR
500 PFN_NUMBER NumberOfPages
;
501 PHYSICAL_MEMORY_RUN Run
[1];
502 } PHYSICAL_MEMORY_DESCRIPTOR
, *PPHYSICAL_MEMORY_DESCRIPTOR
;
504 typedef struct _MMCOLOR_TABLES
509 } MMCOLOR_TABLES
, *PMMCOLOR_TABLES
;
511 typedef struct _MI_LARGE_PAGE_RANGES
513 PFN_NUMBER StartFrame
;
514 PFN_NUMBER LastFrame
;
515 } MI_LARGE_PAGE_RANGES
, *PMI_LARGE_PAGE_RANGES
;
517 typedef struct _MMVIEW
520 PCONTROL_AREA ControlArea
;
523 typedef struct _MMSESSION
525 KGUARDED_MUTEX SystemSpaceViewLock
;
526 PKGUARDED_MUTEX SystemSpaceViewLockPointer
;
527 PCHAR SystemSpaceViewStart
;
528 PMMVIEW SystemSpaceViewTable
;
529 ULONG SystemSpaceHashSize
;
530 ULONG SystemSpaceHashEntries
;
531 ULONG SystemSpaceHashKey
;
532 ULONG BitmapFailures
;
533 PRTL_BITMAP SystemSpaceBitMap
;
534 } MMSESSION
, *PMMSESSION
;
536 typedef struct _MM_SESSION_SPACE_FLAGS
539 ULONG DeletePending
:1;
541 } MM_SESSION_SPACE_FLAGS
;
543 typedef struct _MM_SESSION_SPACE
545 struct _MM_SESSION_SPACE
*GlobalVirtualAddress
;
550 MM_SESSION_SPACE_FLAGS Flags
;
553 LIST_ENTRY ProcessList
;
554 LARGE_INTEGER LastProcessSwappedOutTime
;
555 PFN_NUMBER SessionPageDirectoryIndex
;
556 SIZE_T NonPageablePages
;
557 SIZE_T CommittedPages
;
558 PVOID PagedPoolStart
;
560 PMMPTE PagedPoolBasePde
;
562 LONG ResidentProcessCount
;
563 ULONG SessionPoolAllocationFailures
[4];
564 LIST_ENTRY ImageList
;
568 PEPROCESS LastProcess
;
569 LONG ProcessReferenceToSession
;
570 LIST_ENTRY WsListEntry
;
571 GENERAL_LOOKASIDE Lookaside
[SESSION_POOL_LOOKASIDES
];
573 KGUARDED_MUTEX PagedPoolMutex
;
574 MM_PAGED_POOL_INFO PagedPoolInfo
;
577 PDRIVER_UNLOAD Win32KDriverUnload
;
578 POOL_DESCRIPTOR PagedPool
;
579 #if defined (_M_AMD64)
584 #if defined (_M_AMD64)
585 PMMPTE SpecialPoolFirstPte
;
586 PMMPTE SpecialPoolLastPte
;
587 PMMPTE NextPdeForSpecialPoolExpansion
;
588 PMMPTE LastPdeForSpecialPoolExpansion
;
589 PFN_NUMBER SpecialPagesInUse
;
591 LONG ImageLoadingCount
;
592 } MM_SESSION_SPACE
, *PMM_SESSION_SPACE
;
594 extern PMM_SESSION_SPACE MmSessionSpace
;
595 extern MMPTE HyperTemplatePte
;
596 extern MMPDE ValidKernelPde
;
597 extern MMPTE ValidKernelPte
;
598 extern MMPDE ValidKernelPdeLocal
;
599 extern MMPTE ValidKernelPteLocal
;
600 extern MMPDE DemandZeroPde
;
601 extern MMPTE DemandZeroPte
;
602 extern MMPTE PrototypePte
;
603 extern MMPTE MmDecommittedPte
;
604 extern BOOLEAN MmLargeSystemCache
;
605 extern BOOLEAN MmZeroPageFile
;
606 extern BOOLEAN MmProtectFreedNonPagedPool
;
607 extern BOOLEAN MmTrackLockedPages
;
608 extern BOOLEAN MmTrackPtes
;
609 extern BOOLEAN MmDynamicPfn
;
610 extern BOOLEAN MmMirroring
;
611 extern BOOLEAN MmMakeLowMemory
;
612 extern BOOLEAN MmEnforceWriteProtection
;
613 extern SIZE_T MmAllocationFragment
;
614 extern ULONG MmConsumedPoolPercentage
;
615 extern ULONG MmVerifyDriverBufferType
;
616 extern ULONG MmVerifyDriverLevel
;
617 extern WCHAR MmVerifyDriverBuffer
[512];
618 extern WCHAR MmLargePageDriverBuffer
[512];
619 extern LIST_ENTRY MiLargePageDriverList
;
620 extern BOOLEAN MiLargePageAllDrivers
;
621 extern ULONG MmVerifyDriverBufferLength
;
622 extern ULONG MmLargePageDriverBufferLength
;
623 extern SIZE_T MmSizeOfNonPagedPoolInBytes
;
624 extern SIZE_T MmMaximumNonPagedPoolInBytes
;
625 extern PFN_NUMBER MmMaximumNonPagedPoolInPages
;
626 extern PFN_NUMBER MmSizeOfPagedPoolInPages
;
627 extern PVOID MmNonPagedSystemStart
;
628 extern SIZE_T MiNonPagedSystemSize
;
629 extern PVOID MmNonPagedPoolStart
;
630 extern PVOID MmNonPagedPoolExpansionStart
;
631 extern PVOID MmNonPagedPoolEnd
;
632 extern SIZE_T MmSizeOfPagedPoolInBytes
;
633 extern PVOID MmPagedPoolStart
;
634 extern PVOID MmPagedPoolEnd
;
635 extern PVOID MmSessionBase
;
636 extern SIZE_T MmSessionSize
;
637 extern PMMPTE MmFirstReservedMappingPte
, MmLastReservedMappingPte
;
638 extern PMMPTE MiFirstReservedZeroingPte
;
639 extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes
[2][MmMaximumCacheType
];
640 extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock
;
641 extern SIZE_T MmBootImageSize
;
642 extern PMMPTE MmSystemPtesStart
[MaximumPtePoolTypes
];
643 extern PMMPTE MmSystemPtesEnd
[MaximumPtePoolTypes
];
644 extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor
;
645 extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor
;
646 extern ULONG_PTR MxPfnAllocation
;
647 extern MM_PAGED_POOL_INFO MmPagedPoolInfo
;
648 extern RTL_BITMAP MiPfnBitMap
;
649 extern KGUARDED_MUTEX MmPagedPoolMutex
;
650 extern KGUARDED_MUTEX MmSectionCommitMutex
;
651 extern PVOID MmPagedPoolStart
;
652 extern PVOID MmPagedPoolEnd
;
653 extern PVOID MmNonPagedSystemStart
;
654 extern PVOID MiSystemViewStart
;
655 extern SIZE_T MmSystemViewSize
;
656 extern PVOID MmSessionBase
;
657 extern PVOID MiSessionSpaceEnd
;
658 extern PMMPTE MiSessionImagePteStart
;
659 extern PMMPTE MiSessionImagePteEnd
;
660 extern PMMPTE MiSessionBasePte
;
661 extern PMMPTE MiSessionLastPte
;
662 extern SIZE_T MmSizeOfPagedPoolInBytes
;
663 extern PMMPDE MmSystemPagePtes
;
664 extern PVOID MmSystemCacheStart
;
665 extern PVOID MmSystemCacheEnd
;
666 extern MMSUPPORT MmSystemCacheWs
;
667 extern SIZE_T MmAllocatedNonPagedPool
;
668 extern ULONG MmSpecialPoolTag
;
669 extern PVOID MmHyperSpaceEnd
;
670 extern PMMWSL MmSystemCacheWorkingSetList
;
671 extern SIZE_T MmMinimumNonPagedPoolSize
;
672 extern ULONG MmMinAdditionNonPagedPoolPerMb
;
673 extern SIZE_T MmDefaultMaximumNonPagedPool
;
674 extern ULONG MmMaxAdditionNonPagedPoolPerMb
;
675 extern ULONG MmSecondaryColors
;
676 extern ULONG MmSecondaryColorMask
;
677 extern ULONG MmNumberOfSystemPtes
;
678 extern ULONG MmMaximumNonPagedPoolPercent
;
679 extern ULONG MmLargeStackSize
;
680 extern PMMCOLOR_TABLES MmFreePagesByColor
[FreePageList
+ 1];
681 extern MMPFNLIST MmStandbyPageListByPriority
[8];
682 extern ULONG MmProductType
;
683 extern MM_SYSTEMSIZE MmSystemSize
;
684 extern PKEVENT MiLowMemoryEvent
;
685 extern PKEVENT MiHighMemoryEvent
;
686 extern PKEVENT MiLowPagedPoolEvent
;
687 extern PKEVENT MiHighPagedPoolEvent
;
688 extern PKEVENT MiLowNonPagedPoolEvent
;
689 extern PKEVENT MiHighNonPagedPoolEvent
;
690 extern PFN_NUMBER MmLowMemoryThreshold
;
691 extern PFN_NUMBER MmHighMemoryThreshold
;
692 extern PFN_NUMBER MiLowPagedPoolThreshold
;
693 extern PFN_NUMBER MiHighPagedPoolThreshold
;
694 extern PFN_NUMBER MiLowNonPagedPoolThreshold
;
695 extern PFN_NUMBER MiHighNonPagedPoolThreshold
;
696 extern PFN_NUMBER MmMinimumFreePages
;
697 extern PFN_NUMBER MmPlentyFreePages
;
698 extern SIZE_T MmMinimumStackCommitInBytes
;
699 extern PFN_COUNT MiExpansionPoolPagesInitialCharge
;
700 extern PFN_NUMBER MmResidentAvailablePages
;
701 extern PFN_NUMBER MmResidentAvailableAtInit
;
702 extern ULONG MmTotalFreeSystemPtes
[MaximumPtePoolTypes
];
703 extern PFN_NUMBER MmTotalSystemDriverPages
;
704 extern ULONG MmCritsectTimeoutSeconds
;
705 extern PVOID MiSessionImageStart
;
706 extern PVOID MiSessionImageEnd
;
707 extern PMMPTE MiHighestUserPte
;
708 extern PMMPDE MiHighestUserPde
;
709 extern PFN_NUMBER MmSystemPageDirectory
[PD_COUNT
];
710 extern PMMPTE MmSharedUserDataPte
;
711 extern LIST_ENTRY MmProcessList
;
712 extern BOOLEAN MmZeroingPageThreadActive
;
713 extern KEVENT MmZeroingPageEvent
;
714 extern ULONG MmSystemPageColor
;
715 extern ULONG MmProcessColorSeed
;
716 extern PMMWSL MmWorkingSetList
;
717 extern PFN_NUMBER MiNumberOfFreePages
;
718 extern SIZE_T MmSessionViewSize
;
719 extern SIZE_T MmSessionPoolSize
;
720 extern SIZE_T MmSessionImageSize
;
721 extern PVOID MiSystemViewStart
;
722 extern PVOID MiSessionPoolEnd
; // 0xBE000000
723 extern PVOID MiSessionPoolStart
; // 0xBD000000
724 extern PVOID MiSessionViewStart
; // 0xBE000000
725 extern PVOID MiSessionSpaceWs
;
726 extern ULONG MmMaximumDeadKernelStacks
;
727 extern SLIST_HEADER MmDeadStackSListHead
;
728 extern MM_AVL_TABLE MmSectionBasedRoot
;
729 extern KGUARDED_MUTEX MmSectionBasedMutex
;
730 extern PVOID MmHighSectionBase
;
731 extern SIZE_T MmSystemLockPagesCount
;
732 extern ULONG_PTR MmSubsectionBase
;
733 extern LARGE_INTEGER MmCriticalSectionTimeout
;
734 extern LIST_ENTRY MmWorkingSetExpansionHead
;
738 MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType
)
740 return ((MemoryType
== LoaderFree
) ||
741 (MemoryType
== LoaderLoadedProgram
) ||
742 (MemoryType
== LoaderFirmwareTemporary
) ||
743 (MemoryType
== LoaderOsloaderStack
));
748 MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType
)
750 return ((MemoryType
== LoaderFirmwarePermanent
) ||
751 (MemoryType
== LoaderSpecialMemory
) ||
752 (MemoryType
== LoaderHALCachedMemory
) ||
753 (MemoryType
== LoaderBBTMemory
));
759 MiIsUserPxe(PVOID Address
)
761 return ((ULONG_PTR
)Address
>> 7) == 0x1FFFFEDF6FB7DA0ULL
;
766 MiIsUserPpe(PVOID Address
)
768 return ((ULONG_PTR
)Address
>> 16) == 0xFFFFF6FB7DA0ULL
;
773 MiIsUserPde(PVOID Address
)
775 return ((ULONG_PTR
)Address
>> 25) == 0x7FFFFB7DA0ULL
;
780 MiIsUserPte(PVOID Address
)
782 return ((ULONG_PTR
)Address
>> 34) == 0x3FFFFDA0ULL
;
787 MiIsUserPde(PVOID Address
)
789 return ((Address
>= (PVOID
)MiAddressToPde(NULL
)) &&
790 (Address
<= (PVOID
)MiHighestUserPde
));
795 MiIsUserPte(PVOID Address
)
797 return (Address
<= (PVOID
)MiHighestUserPte
);
802 // Figures out the hardware bits for a PTE
806 MiDetermineUserGlobalPteMask(IN PVOID PointerPte
)
813 /* Make it valid and accessed */
814 TempPte
.u
.Hard
.Valid
= TRUE
;
815 MI_MAKE_ACCESSED_PAGE(&TempPte
);
817 /* Is this for user-mode? */
819 #if (_MI_PAGING_LEVELS == 4)
820 MiIsUserPxe(PointerPte
) ||
822 #if (_MI_PAGING_LEVELS >= 3)
823 MiIsUserPpe(PointerPte
) ||
825 MiIsUserPde(PointerPte
) ||
826 MiIsUserPte(PointerPte
))
828 /* Set the owner bit */
829 MI_MAKE_OWNER_PAGE(&TempPte
);
832 /* FIXME: We should also set the global bit */
834 /* Return the protection */
835 return TempPte
.u
.Long
;
839 // Creates a valid kernel PTE with the given protection
843 MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte
,
844 IN PMMPTE MappingPte
,
845 IN ULONG_PTR ProtectionMask
,
846 IN PFN_NUMBER PageFrameNumber
)
848 /* Only valid for kernel, non-session PTEs */
849 ASSERT(MappingPte
> MiHighestUserPte
);
850 ASSERT(!MI_IS_SESSION_PTE(MappingPte
));
851 ASSERT((MappingPte
< (PMMPTE
)PDE_BASE
) || (MappingPte
> (PMMPTE
)PDE_TOP
));
854 *NewPte
= ValidKernelPte
;
856 /* Set the protection and page */
857 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
858 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
862 // Creates a valid PTE with the given protection
866 MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte
,
867 IN PMMPTE MappingPte
,
868 IN ULONG_PTR ProtectionMask
,
869 IN PFN_NUMBER PageFrameNumber
)
871 /* Set the protection and page */
872 NewPte
->u
.Long
= MiDetermineUserGlobalPteMask(MappingPte
);
873 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
874 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
878 // Creates a valid user PTE with the given protection
882 MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte
,
883 IN PMMPTE MappingPte
,
884 IN ULONG_PTR ProtectionMask
,
885 IN PFN_NUMBER PageFrameNumber
)
887 /* Only valid for kernel, non-session PTEs */
888 ASSERT(MappingPte
<= MiHighestUserPte
);
893 /* Set the protection and page */
894 NewPte
->u
.Hard
.Valid
= TRUE
;
895 NewPte
->u
.Hard
.Owner
= TRUE
;
896 NewPte
->u
.Hard
.PageFrameNumber
= PageFrameNumber
;
897 NewPte
->u
.Long
|= MmProtectToPteMask
[ProtectionMask
];
902 // Builds a Prototype PTE for the address of the PTE
906 MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte
,
907 IN PMMPTE PointerPte
)
911 /* Mark this as a prototype */
913 NewPte
->u
.Proto
.Prototype
= 1;
916 * Prototype PTEs are only valid in paged pool by design, this little trick
917 * lets us only use 30 bits for the adress of the PTE, as long as the area
918 * stays 1024MB At most.
920 Offset
= (ULONG_PTR
)PointerPte
- (ULONG_PTR
)MmPagedPoolStart
;
923 * 7 bits go in the "low" (but we assume the bottom 2 are zero)
924 * and the other 21 bits go in the "high"
926 NewPte
->u
.Proto
.ProtoAddressLow
= (Offset
& 0x1FC) >> 2;
927 NewPte
->u
.Proto
.ProtoAddressHigh
= (Offset
& 0x3FFFFE00) >> 9;
931 // Builds a Subsection PTE for the address of the Segment
935 MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte
,
940 /* Mark this as a prototype */
942 NewPte
->u
.Subsect
.Prototype
= 1;
945 * Segments are only valid either in nonpaged pool. We store the 20 bit
946 * difference either from the top or bottom of nonpaged pool, giving a
947 * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed
950 if ((ULONG_PTR
)Segment
< ((ULONG_PTR
)MmSubsectionBase
+ (128 * _1MB
)))
952 Offset
= (ULONG_PTR
)Segment
- (ULONG_PTR
)MmSubsectionBase
;
953 NewPte
->u
.Subsect
.WhichPool
= PagedPool
;
957 Offset
= (ULONG_PTR
)MmNonPagedPoolEnd
- (ULONG_PTR
)Segment
;
958 NewPte
->u
.Subsect
.WhichPool
= NonPagedPool
;
962 * 4 bits go in the "low" (but we assume the bottom 3 are zero)
963 * and the other 20 bits go in the "high"
965 NewPte
->u
.Subsect
.SubsectionAddressLow
= (Offset
& 0x78) >> 3;
966 NewPte
->u
.Subsect
.SubsectionAddressHigh
= (Offset
& 0xFFFFF80) >> 7;
971 MI_IS_MAPPED_PTE(PMMPTE PointerPte
)
973 /// \todo Make this reasonable code, this is UGLY!
974 return ((PointerPte
->u
.Long
& 0xFFFFFC01) != 0);
980 // Returns if the page is physically resident (ie: a large page)
981 // FIXFIX: CISC/x86 only?
985 MI_IS_PHYSICAL_ADDRESS(IN PVOID Address
)
989 /* Large pages are never paged out, always physically resident */
990 PointerPde
= MiAddressToPde(Address
);
991 return ((PointerPde
->u
.Hard
.LargePage
) && (PointerPde
->u
.Hard
.Valid
));
995 // Writes a valid PTE
999 MI_WRITE_VALID_PTE(IN PMMPTE PointerPte
,
1002 /* Write the valid PTE */
1003 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1004 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
1005 *PointerPte
= TempPte
;
1009 // Updates a valid PTE
1013 MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte
,
1016 /* Write the valid PTE */
1017 ASSERT(PointerPte
->u
.Hard
.Valid
== 1);
1018 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
1019 ASSERT(PointerPte
->u
.Hard
.PageFrameNumber
== TempPte
.u
.Hard
.PageFrameNumber
);
1020 *PointerPte
= TempPte
;
1024 // Writes an invalid PTE
1028 MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte
,
1029 IN MMPTE InvalidPte
)
1031 /* Write the invalid PTE */
1032 ASSERT(InvalidPte
.u
.Hard
.Valid
== 0);
1033 ASSERT(InvalidPte
.u
.Long
!= 0);
1034 *PointerPte
= InvalidPte
;
1038 // Erase the PTE completely
1042 MI_ERASE_PTE(IN PMMPTE PointerPte
)
1044 /* Zero out the PTE */
1045 ASSERT(PointerPte
->u
.Long
!= 0);
1046 PointerPte
->u
.Long
= 0;
1050 // Writes a valid PDE
1054 MI_WRITE_VALID_PDE(IN PMMPDE PointerPde
,
1057 /* Write the valid PDE */
1058 ASSERT(PointerPde
->u
.Hard
.Valid
== 0);
1059 ASSERT(TempPde
.u
.Hard
.Valid
== 1);
1060 *PointerPde
= TempPde
;
1064 // Writes an invalid PDE
1068 MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde
,
1069 IN MMPDE InvalidPde
)
1071 /* Write the invalid PDE */
1072 ASSERT(InvalidPde
.u
.Hard
.Valid
== 0);
1073 ASSERT(InvalidPde
.u
.Long
!= 0);
1074 *PointerPde
= InvalidPde
;
1078 // Checks if the thread already owns a working set
1082 MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread
)
1084 /* If any of these are held, return TRUE */
1085 return ((Thread
->OwnsProcessWorkingSetExclusive
) ||
1086 (Thread
->OwnsProcessWorkingSetShared
) ||
1087 (Thread
->OwnsSystemWorkingSetExclusive
) ||
1088 (Thread
->OwnsSystemWorkingSetShared
) ||
1089 (Thread
->OwnsSessionWorkingSetExclusive
) ||
1090 (Thread
->OwnsSessionWorkingSetShared
));
1094 // Checks if the process owns the working set lock
1098 MI_WS_OWNER(IN PEPROCESS Process
)
1100 /* Check if this process is the owner, and that the thread owns the WS */
1101 if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive
== 0)
1103 DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread());
1105 if (KeGetCurrentThread()->ApcState
.Process
!= &Process
->Pcb
)
1107 DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process
);
1109 return ((KeGetCurrentThread()->ApcState
.Process
== &Process
->Pcb
) &&
1110 ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive
) ||
1111 (PsGetCurrentThread()->OwnsProcessWorkingSetShared
)));
1115 // New ARM3<->RosMM PAGE Architecture
1119 MiIsRosSectionObject(IN PVOID Section
)
1121 PROS_SECTION_OBJECT RosSection
= Section
;
1122 if ((RosSection
->Type
== 'SC') && (RosSection
->Size
== 'TN')) return TRUE
;
1126 #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE)
1130 MiDecrementReferenceCount(
1132 IN PFN_NUMBER PageFrameIndex
1137 MI_IS_WS_UNSAFE(IN PEPROCESS Process
)
1139 return (Process
->Vm
.Flags
.AcquiredUnsafe
== TRUE
);
1143 // Locks the working set for the given process
1147 MiLockProcessWorkingSet(IN PEPROCESS Process
,
1150 /* Shouldn't already be owning the process working set */
1151 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1152 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1154 /* Block APCs, make sure that still nothing is already held */
1155 KeEnterGuardedRegion();
1156 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1158 /* Lock the working set */
1159 ExAcquirePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1161 /* Now claim that we own the lock */
1162 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1163 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1164 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1169 MiLockProcessWorkingSetShared(IN PEPROCESS Process
,
1172 /* Shouldn't already be owning the process working set */
1173 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1174 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1176 /* Block APCs, make sure that still nothing is already held */
1177 KeEnterGuardedRegion();
1178 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1180 /* Lock the working set */
1181 ExAcquirePushLockShared(&Process
->Vm
.WorkingSetMutex
);
1183 /* Now claim that we own the lock */
1184 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1185 ASSERT(Thread
->OwnsProcessWorkingSetShared
== FALSE
);
1186 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1187 Thread
->OwnsProcessWorkingSetShared
= TRUE
;
1192 MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process
,
1195 /* Shouldn't already be owning the process working set */
1196 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1198 /* APCs must be blocked, make sure that still nothing is already held */
1199 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1200 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1202 /* Lock the working set */
1203 ExAcquirePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1205 /* Now claim that we own the lock */
1206 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1207 Process
->Vm
.Flags
.AcquiredUnsafe
= 1;
1208 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1209 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1213 // Unlocks the working set for the given process
1217 MiUnlockProcessWorkingSet(IN PEPROCESS Process
,
1220 /* Make sure we are the owner of a safe acquisition */
1221 ASSERT(MI_WS_OWNER(Process
));
1222 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1224 /* The thread doesn't own it anymore */
1225 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== TRUE
);
1226 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1228 /* Release the lock and re-enable APCs */
1229 ExReleasePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1230 KeLeaveGuardedRegion();
1234 // Unlocks the working set for the given process
1238 MiUnlockProcessWorkingSetShared(IN PEPROCESS Process
,
1241 /* Make sure we are the owner of a safe acquisition (because shared) */
1242 ASSERT(MI_WS_OWNER(Process
));
1243 ASSERT(!MI_IS_WS_UNSAFE(Process
));
1245 /* Ensure we are in a shared acquisition */
1246 ASSERT(Thread
->OwnsProcessWorkingSetShared
== TRUE
);
1247 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== FALSE
);
1249 /* Don't claim the lock anylonger */
1250 Thread
->OwnsProcessWorkingSetShared
= FALSE
;
1252 /* Release the lock and re-enable APCs */
1253 ExReleasePushLockShared(&Process
->Vm
.WorkingSetMutex
);
1254 KeLeaveGuardedRegion();
1258 // Unlocks the working set for the given process
1262 MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process
,
1265 /* Make sure we are the owner of an unsafe acquisition */
1266 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1267 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1268 ASSERT(MI_WS_OWNER(Process
));
1269 ASSERT(MI_IS_WS_UNSAFE(Process
));
1271 /* No longer unsafe */
1272 Process
->Vm
.Flags
.AcquiredUnsafe
= 0;
1274 /* The thread doesn't own it anymore */
1275 ASSERT(Thread
->OwnsProcessWorkingSetExclusive
== TRUE
);
1276 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1278 /* Release the lock but don't touch APC state */
1279 ExReleasePushLockExclusive(&Process
->Vm
.WorkingSetMutex
);
1280 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1284 // Locks the working set
1288 MiLockWorkingSet(IN PETHREAD Thread
,
1289 IN PMMSUPPORT WorkingSet
)
1292 KeEnterGuardedRegion();
1294 /* Working set should be in global memory */
1295 ASSERT(MI_IS_SESSION_ADDRESS((PVOID
)WorkingSet
) == FALSE
);
1297 /* Thread shouldn't already be owning something */
1298 ASSERT(!MM_ANY_WS_LOCK_HELD(Thread
));
1300 /* Lock this working set */
1301 ExAcquirePushLockExclusive(&WorkingSet
->WorkingSetMutex
);
1303 /* Which working set is this? */
1304 if (WorkingSet
== &MmSystemCacheWs
)
1306 /* Own the system working set */
1307 ASSERT((Thread
->OwnsSystemWorkingSetExclusive
== FALSE
) &&
1308 (Thread
->OwnsSystemWorkingSetShared
== FALSE
));
1309 Thread
->OwnsSystemWorkingSetExclusive
= TRUE
;
1311 else if (WorkingSet
->Flags
.SessionSpace
)
1313 /* Own the session working set */
1314 ASSERT((Thread
->OwnsSessionWorkingSetExclusive
== FALSE
) &&
1315 (Thread
->OwnsSessionWorkingSetShared
== FALSE
));
1316 Thread
->OwnsSessionWorkingSetExclusive
= TRUE
;
1320 /* Own the process working set */
1321 ASSERT((Thread
->OwnsProcessWorkingSetExclusive
== FALSE
) &&
1322 (Thread
->OwnsProcessWorkingSetShared
== FALSE
));
1323 Thread
->OwnsProcessWorkingSetExclusive
= TRUE
;
1328 // Unlocks the working set
1332 MiUnlockWorkingSet(IN PETHREAD Thread
,
1333 IN PMMSUPPORT WorkingSet
)
1335 /* Working set should be in global memory */
1336 ASSERT(MI_IS_SESSION_ADDRESS((PVOID
)WorkingSet
) == FALSE
);
1338 /* Which working set is this? */
1339 if (WorkingSet
== &MmSystemCacheWs
)
1341 /* Release the system working set */
1342 ASSERT((Thread
->OwnsSystemWorkingSetExclusive
== TRUE
) ||
1343 (Thread
->OwnsSystemWorkingSetShared
== TRUE
));
1344 Thread
->OwnsSystemWorkingSetExclusive
= FALSE
;
1346 else if (WorkingSet
->Flags
.SessionSpace
)
1348 /* Release the session working set */
1349 ASSERT((Thread
->OwnsSessionWorkingSetExclusive
== TRUE
) ||
1350 (Thread
->OwnsSessionWorkingSetShared
== TRUE
));
1351 Thread
->OwnsSessionWorkingSetExclusive
= 0;
1355 /* Release the process working set */
1356 ASSERT((Thread
->OwnsProcessWorkingSetExclusive
) ||
1357 (Thread
->OwnsProcessWorkingSetShared
));
1358 Thread
->OwnsProcessWorkingSetExclusive
= FALSE
;
1361 /* Release the working set lock */
1362 ExReleasePushLockExclusive(&WorkingSet
->WorkingSetMutex
);
1365 KeLeaveGuardedRegion();
1370 MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process
,
1373 OUT PBOOLEAN Shared
)
1375 ASSERT(MI_WS_OWNER(Process
));
1377 /* Check if the current owner is unsafe */
1378 if (MI_IS_WS_UNSAFE(Process
))
1380 /* Release unsafely */
1381 MiUnlockProcessWorkingSetUnsafe(Process
, Thread
);
1385 else if (Thread
->OwnsProcessWorkingSetExclusive
== 1)
1387 /* Owner is safe and exclusive, release normally */
1388 MiUnlockProcessWorkingSet(Process
, Thread
);
1394 /* Owner is shared (implies safe), release normally */
1395 MiUnlockProcessWorkingSetShared(Process
, Thread
);
1403 MiLockProcessWorkingSetForFault(IN PEPROCESS Process
,
1408 /* Check if this was a safe lock or not */
1413 /* Reacquire safely & shared */
1414 MiLockProcessWorkingSetShared(Process
, Thread
);
1418 /* Reacquire safely */
1419 MiLockProcessWorkingSet(Process
, Thread
);
1424 /* Unsafe lock cannot be shared */
1425 ASSERT(Shared
== FALSE
);
1426 /* Reacquire unsafely */
1427 MiLockProcessWorkingSetUnsafe(Process
, Thread
);
1432 // Returns the ProtoPTE inside a VAD for the given VPN
1436 MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad
,
1441 /* Find the offset within the VAD's prototype PTEs */
1442 ProtoPte
= Vad
->FirstPrototypePte
+ (Vpn
- Vad
->StartingVpn
);
1443 ASSERT(ProtoPte
<= Vad
->LastContiguousPte
);
1448 // Returns the PFN Database entry for the given page number
1449 // Warning: This is not necessarily a valid PFN database entry!
1453 MI_PFN_ELEMENT(IN PFN_NUMBER Pfn
)
1456 return &MmPfnDatabase
[Pfn
];
1460 // Drops a locked page without dereferencing it
1464 MiDropLockCount(IN PMMPFN Pfn1
)
1466 /* This page shouldn't be locked, but it should be valid */
1467 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
1468 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1470 /* Is this the last reference to the page */
1471 if (Pfn1
->u3
.e2
.ReferenceCount
== 1)
1473 /* It better not be valid */
1474 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1476 /* Is it a prototype PTE? */
1477 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1478 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1480 /* FIXME: We should return commit */
1481 DPRINT1("Not returning commit for prototype PTE\n");
1484 /* Update the counter */
1485 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1490 // Drops a locked page and dereferences it
1494 MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1
)
1496 USHORT RefCount
, OldRefCount
;
1497 PFN_NUMBER PageFrameIndex
;
1499 /* Loop while we decrement the page successfully */
1502 /* There should be at least one reference */
1503 OldRefCount
= Pfn1
->u3
.e2
.ReferenceCount
;
1504 ASSERT(OldRefCount
!= 0);
1506 /* Are we the last one */
1507 if (OldRefCount
== 1)
1509 /* The page shoudln't be shared not active at this point */
1510 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 1);
1511 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1512 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1514 /* Is it a prototype PTE? */
1515 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1516 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1518 /* FIXME: We should return commit */
1519 DPRINT1("Not returning commit for prototype PTE\n");
1522 /* Update the counter, and drop a reference the long way */
1523 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1524 PageFrameIndex
= MiGetPfnEntryIndex(Pfn1
);
1525 MiDecrementReferenceCount(Pfn1
, PageFrameIndex
);
1529 /* Drop a reference the short way, and that's it */
1530 RefCount
= InterlockedCompareExchange16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
,
1533 ASSERT(RefCount
!= 0);
1534 } while (OldRefCount
!= RefCount
);
1536 /* If we got here, there should be more than one reference */
1537 ASSERT(RefCount
> 1);
1540 /* Is it still being shared? */
1541 if (Pfn1
->u2
.ShareCount
>= 1)
1543 /* Then it should be valid */
1544 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1546 /* Is it a prototype PTE? */
1547 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1548 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1550 /* We don't handle ethis */
1554 /* Update the counter */
1555 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1561 // References a locked page and updates the counter
1562 // Used in MmProbeAndLockPages to handle different edge cases
1566 MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1568 USHORT RefCount
, OldRefCount
;
1571 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
1573 /* Does ARM3 own the page? */
1574 if (MI_IS_ROS_PFN(Pfn1
))
1576 /* ReactOS Mm doesn't track share count */
1577 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1581 /* On ARM3 pages, we should see a valid share count */
1582 ASSERT((Pfn1
->u2
.ShareCount
!= 0) && (Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
));
1584 /* Is it a prototype PTE? */
1585 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1586 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1588 /* FIXME: We should charge commit */
1589 DPRINT1("Not charging commit for prototype PTE\n");
1593 /* More locked pages! */
1594 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1596 /* Loop trying to update the reference count */
1599 /* Get the current reference count, make sure it's valid */
1600 OldRefCount
= Pfn1
->u3
.e2
.ReferenceCount
;
1601 ASSERT(OldRefCount
!= 0);
1602 ASSERT(OldRefCount
< 2500);
1604 /* Bump it up by one */
1605 RefCount
= InterlockedCompareExchange16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
,
1608 ASSERT(RefCount
!= 0);
1609 } while (OldRefCount
!= RefCount
);
1611 /* Was this the first lock attempt? If not, undo our bump */
1612 if (OldRefCount
!= 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1616 // References a locked page and updates the counter
1617 // Used in all other cases except MmProbeAndLockPages
1621 MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1625 /* Is it a prototype PTE? */
1626 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1627 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1629 /* FIXME: We should charge commit */
1630 DPRINT1("Not charging commit for prototype PTE\n");
1633 /* More locked pages! */
1634 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1636 /* Update the reference count */
1637 NewRefCount
= InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1638 if (NewRefCount
== 2)
1640 /* Is it locked or shared? */
1641 if (Pfn1
->u2
.ShareCount
)
1643 /* It's shared, so make sure it's active */
1644 ASSERT(Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
);
1648 /* It's locked, so we shouldn't lock again */
1649 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1654 /* Someone had already locked the page, so undo our bump */
1655 ASSERT(NewRefCount
< 2500);
1656 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1661 // References a locked page and updates the counter
1662 // Used in all other cases except MmProbeAndLockPages
1666 MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1
)
1670 /* Make sure the page isn't used yet */
1671 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1672 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1674 /* Is it a prototype PTE? */
1675 if ((Pfn1
->u3
.e1
.PrototypePte
== 1) &&
1676 (Pfn1
->OriginalPte
.u
.Soft
.Prototype
== 1))
1678 /* FIXME: We should charge commit */
1679 DPRINT1("Not charging commit for prototype PTE\n");
1682 /* More locked pages! */
1683 InterlockedIncrementSizeT(&MmSystemLockPagesCount
);
1685 /* Update the reference count */
1686 NewRefCount
= InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1687 if (NewRefCount
!= 1)
1689 /* Someone had already locked the page, so undo our bump */
1690 ASSERT(NewRefCount
< 2500);
1691 InterlockedDecrementSizeT(&MmSystemLockPagesCount
);
1697 MiIncrementPageTableReferences(IN PVOID Address
)
1701 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1704 ASSERT(*RefCount
<= PTE_PER_PAGE
);
1709 MiDecrementPageTableReferences(IN PVOID Address
)
1713 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1716 ASSERT(*RefCount
< PTE_PER_PAGE
);
1721 MiQueryPageTableReferences(IN PVOID Address
)
1725 RefCount
= &MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)];
1734 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1739 MiInitializeSessionSpaceLayout();
1743 MiInitMachineDependent(
1744 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1749 MiComputeColorInformation(
1756 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1761 MiInitializeColorTables(
1767 MiInitializePfnDatabase(
1768 IN PLOADER_PARAMETER_BLOCK LoaderBlock
1773 MiInitializeSessionWsSupport(
1779 MiInitializeSessionIds(
1785 MiInitializeMemoryEvents(
1792 IN PFN_NUMBER PageCount
1795 PPHYSICAL_MEMORY_DESCRIPTOR
1797 MmInitializeMemoryLimits(
1798 IN PLOADER_PARAMETER_BLOCK LoaderBlock
,
1799 IN PBOOLEAN IncludeType
1804 MiPagesInLoaderBlock(
1805 IN PLOADER_PARAMETER_BLOCK LoaderBlock
,
1806 IN PBOOLEAN IncludeType
1812 IN PVOID AddressStart
,
1818 MiRosProtectVirtualMemory(
1819 IN PEPROCESS Process
,
1820 IN OUT PVOID
*BaseAddress
,
1821 IN OUT PSIZE_T NumberOfBytesToProtect
,
1822 IN ULONG NewAccessProtection
,
1823 OUT PULONG OldAccessProtection OPTIONAL
1829 IN BOOLEAN StoreInstruction
,
1831 IN KPROCESSOR_MODE Mode
,
1832 IN PVOID TrapInformation
1837 MiCheckPdeForPagedPool(
1843 MiInitializeNonPagedPool(
1849 MiInitializeNonPagedPoolThresholds(
1855 MiInitializePoolEvents(
1862 IN POOL_TYPE PoolType
,// FIXFIX: This should go in ex.h after the pool merge
1863 IN ULONG Threshold
//
1866 // FIXFIX: THIS ONE TOO
1870 ExInitializePoolDescriptor(
1871 IN PPOOL_DESCRIPTOR PoolDescriptor
,
1872 IN POOL_TYPE PoolType
,
1880 MiInitializeSessionPool(
1886 MiInitializeSystemPtes(
1887 IN PMMPTE StartingPte
,
1888 IN ULONG NumberOfPtes
,
1889 IN MMSYSTEM_PTE_POOL_TYPE PoolType
1894 MiReserveSystemPtes(
1895 IN ULONG NumberOfPtes
,
1896 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1901 MiReleaseSystemPtes(
1902 IN PMMPTE StartingPte
,
1903 IN ULONG NumberOfPtes
,
1904 IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
1910 MiFindContiguousPages(
1911 IN PFN_NUMBER LowestPfn
,
1912 IN PFN_NUMBER HighestPfn
,
1913 IN PFN_NUMBER BoundaryPfn
,
1914 IN PFN_NUMBER SizeInPages
,
1915 IN MEMORY_CACHING_TYPE CacheType
1920 MiCheckForContiguousMemory(
1921 IN PVOID BaseAddress
,
1922 IN PFN_NUMBER BaseAddressPages
,
1923 IN PFN_NUMBER SizeInPages
,
1924 IN PFN_NUMBER LowestPfn
,
1925 IN PFN_NUMBER HighestPfn
,
1926 IN PFN_NUMBER BoundaryPfn
,
1927 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
1932 MiAllocatePagesForMdl(
1933 IN PHYSICAL_ADDRESS LowAddress
,
1934 IN PHYSICAL_ADDRESS HighAddress
,
1935 IN PHYSICAL_ADDRESS SkipBytes
,
1936 IN SIZE_T TotalBytes
,
1937 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
,
1943 MiMapLockedPagesInUserSpace(
1946 IN MEMORY_CACHING_TYPE CacheType
,
1947 IN PVOID BaseAddress
1952 MiUnmapLockedPagesInUserSpace(
1953 IN PVOID BaseAddress
,
1960 IN PMMPFNLIST ListHead
,
1961 IN PFN_NUMBER PageFrameIndex
1966 MiUnlinkFreeOrZeroedPage(
1972 MiUnlinkPageFromList(
1979 IN PFN_NUMBER PageFrameIndex
,
1980 IN PMMPTE PointerPte
,
1986 MiInitializeAndChargePfn(
1987 OUT PPFN_NUMBER PageFrameIndex
,
1988 IN PMMPTE PointerPde
,
1989 IN PFN_NUMBER ContainingPageFrame
,
1990 IN BOOLEAN SessionAllocation
1995 MiInitializePfnAndMakePteValid(
1996 IN PFN_NUMBER PageFrameIndex
,
1997 IN PMMPTE PointerPte
,
2003 MiInitializePfnForOtherProcess(
2004 IN PFN_NUMBER PageFrameIndex
,
2005 IN PMMPTE PointerPte
,
2006 IN PFN_NUMBER PteFrame
2011 MiDecrementShareCount(
2013 IN PFN_NUMBER PageFrameIndex
2031 IN PFN_NUMBER PageFrameIndex
2036 MiInsertPageInFreeList(
2037 IN PFN_NUMBER PageFrameIndex
2042 MiDeleteSystemPageableVm(
2043 IN PMMPTE PointerPte
,
2044 IN PFN_NUMBER PageCount
,
2046 OUT PPFN_NUMBER ValidPages
2051 MiGetPageProtection(
2052 IN PMMPTE PointerPte
2055 PLDR_DATA_TABLE_ENTRY
2057 MiLookupDataTableEntry(
2063 MiInitializeDriverLargePageList(
2069 MiInitializeLargePageSupport(
2088 IN PVOID VirtualAddress
2093 MiCheckForConflictingNode(
2094 IN ULONG_PTR StartVpn
,
2095 IN ULONG_PTR EndVpn
,
2096 IN PMM_AVL_TABLE Table
,
2097 OUT PMMADDRESS_NODE
*NodeOrParent
2102 MiFindEmptyAddressRangeDownTree(
2104 IN ULONG_PTR BoundaryAddress
,
2105 IN ULONG_PTR Alignment
,
2106 IN PMM_AVL_TABLE Table
,
2107 OUT PULONG_PTR Base
,
2108 OUT PMMADDRESS_NODE
*Parent
2113 MiFindEmptyAddressRangeDownBasedTree(
2115 IN ULONG_PTR BoundaryAddress
,
2116 IN ULONG_PTR Alignment
,
2117 IN PMM_AVL_TABLE Table
,
2123 MiFindEmptyAddressRangeInTree(
2125 IN ULONG_PTR Alignment
,
2126 IN PMM_AVL_TABLE Table
,
2127 OUT PMMADDRESS_NODE
*PreviousVad
,
2137 IN ULONG ProtectionMask
2144 IN PEPROCESS Process
2151 _In_ ULONG_PTR
*BaseAddress
,
2152 _In_ SIZE_T ViewSize
,
2153 _In_ ULONG_PTR HighestAddress
,
2154 _In_ ULONG_PTR Alignment
,
2155 _In_ ULONG AllocationType
);
2159 MiInsertBasedSection(
2165 MiUnmapViewOfSection(
2166 IN PEPROCESS Process
,
2167 IN PVOID BaseAddress
,
2173 MiRosUnmapViewOfSection(
2174 IN PEPROCESS Process
,
2175 IN PVOID BaseAddress
,
2182 IN PMM_AVL_TABLE Table
,
2183 IN PMMADDRESS_NODE NewNode
,
2184 PMMADDRESS_NODE Parent
,
2185 TABLE_SEARCH_RESULT Result
2191 IN PMMADDRESS_NODE Node
,
2192 IN PMM_AVL_TABLE Table
2198 IN PMMADDRESS_NODE Node
2204 IN PMMADDRESS_NODE Node
2209 MiInitializeSystemSpaceMap(
2210 IN PMMSESSION InputSession OPTIONAL
2215 MiSessionRemoveProcess(
2221 MiReleaseProcessReferenceToSessionDataPage(
2222 IN PMM_SESSION_SPACE SessionGlobal
2227 MiSessionAddProcess(
2228 IN PEPROCESS NewProcess
2233 MiSessionCommitPageTables(
2240 MiMakeProtectionMask(
2246 MiDeleteVirtualAddresses(
2248 IN ULONG_PTR EndingAddress
,
2254 MiMakeSystemAddressValid(
2255 IN PVOID PageTableVirtualAddress
,
2256 IN PEPROCESS CurrentProcess
2261 MiMakeSystemAddressValidPfn(
2262 IN PVOID VirtualAddress
,
2269 IN PEPROCESS CurrentProcess
,
2282 MiDeleteARM3Section(
2288 MiQueryMemorySectionName(
2289 IN HANDLE ProcessHandle
,
2290 IN PVOID BaseAddress
,
2291 OUT PVOID MemoryInformation
,
2292 IN SIZE_T MemoryInformationLength
,
2293 OUT PSIZE_T ReturnLength
2298 MiRosUnmapViewInSystemSpace(
2304 MmDeterminePoolType(
2305 IN PVOID PoolAddress
2310 MiMakePdeExistAndMakeValid(
2311 IN PMMPTE PointerPde
,
2312 IN PEPROCESS TargetProcess
,
2317 // MiRemoveZeroPage will use inline code to zero out the page manually if only
2318 // free pages are available. In some scenarios, we don't/can't run that piece of
2319 // code and would rather only have a real zero page. If we can't have a zero page,
2320 // then we'd like to have our own code to grab a free page and zero it out, by
2321 // using MiRemoveAnyPage. This macro implements this.
2325 MiRemoveZeroPageSafe(IN ULONG Color
)
2327 if (MmFreePagesByColor
[ZeroedPageList
][Color
].Flink
!= LIST_HEAD
) return MiRemoveZeroPage(Color
);