2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
11 /* INCLUDES ****************************************************************/
17 #if defined (ALLOC_PRAGMA)
18 #pragma alloc_text(INIT, MmInitializePageList)
21 #define MODULE_INVOLVED_IN_ARM3
22 #include "ARM3/miarm.h"
24 /* TYPES *******************************************************************/
26 #define MM_PHYSICAL_PAGE_FREE (0x1)
27 #define MM_PHYSICAL_PAGE_USED (0x2)
29 /* GLOBALS ****************************************************************/
33 // ReactOS to NT Physical Page Descriptor Entry Legacy Mapping Definitions
42 LIST_ENTRY ListEntry
; // 0x000
43 ULONG_PTR RmapListHead
; // 0x008
44 USHORT ReferenceCount
; // 0x00C
48 USHORT StartOfAllocation
:1;
49 USHORT EndOfAllocation
:1;
58 LONG MapCount
; // 0x10
59 ULONG_PTR SavedSwapEntry
; // 0x018
61 } PHYSICAL_PAGE
, *PPHYSICAL_PAGE
;
63 C_ASSERT(sizeof(PHYSICAL_PAGE
) == sizeof(MMPFN
));
65 #define MiGetPfnEntry(Pfn) ((PPHYSICAL_PAGE)MiGetPfnEntry(Pfn))
66 #define MiGetPfnEntryIndex(x) MiGetPfnEntryIndex((struct _MMPFN*)x)
67 #define LockCount Flags.LockCount
70 #define MmPfnDatabase ((PPHYSICAL_PAGE)MmPfnDatabase)
72 #define MMPFN PHYSICAL_PAGE
73 #define PMMPFN PPHYSICAL_PAGE
75 ULONG MmAvailablePages
;
76 ULONG MmResidentAvailablePages
;
78 SIZE_T MmTotalCommitLimit
;
79 SIZE_T MmTotalCommittedPages
;
80 SIZE_T MmSharedCommit
;
81 SIZE_T MmDriverCommit
;
82 SIZE_T MmProcessCommit
;
83 SIZE_T MmPagedPoolCommit
;
84 SIZE_T MmPeakCommitment
;
85 SIZE_T MmtotalCommitLimitMaximum
;
87 MMPFNLIST MmZeroedPageListHead
;
88 MMPFNLIST MmFreePageListHead
;
89 MMPFNLIST MmStandbyPageListHead
;
90 MMPFNLIST MmModifiedPageListHead
;
91 MMPFNLIST MmModifiedNoWritePageListHead
;
93 /* List of pages allocated to the MC_USER Consumer */
94 static LIST_ENTRY UserPageListHead
;
95 /* List of pages zeroed by the ZPW (MmZeroPageThreadMain) */
96 static LIST_ENTRY FreeZeroedPageListHead
;
97 /* List of free pages, filled by MmGetReferenceCountPage and
98 * and MmInitializePageList */
99 static LIST_ENTRY FreeUnzeroedPageListHead
;
101 static KEVENT ZeroPageThreadEvent
;
102 static BOOLEAN ZeroPageThreadShouldTerminate
= FALSE
;
104 static ULONG UnzeroedPageCount
= 0;
106 /* FUNCTIONS *************************************************************/
110 MmGetLRUFirstUserPage(VOID
)
112 PLIST_ENTRY NextListEntry
;
113 PHYSICAL_PAGE
* PageDescriptor
;
116 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
117 NextListEntry
= UserPageListHead
.Flink
;
118 if (NextListEntry
== &UserPageListHead
)
120 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
123 PageDescriptor
= CONTAINING_RECORD(NextListEntry
, PHYSICAL_PAGE
, ListEntry
);
124 ASSERT_PFN(&PageDescriptor
->Pfn
);
125 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
126 return PageDescriptor
- MmPfnDatabase
;
131 MmInsertLRULastUserPage(PFN_NUMBER Pfn
)
136 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
137 Page
= MiGetPfnEntry(Pfn
);
139 ASSERT(Page
->Flags
.Type
== MM_PHYSICAL_PAGE_USED
);
140 ASSERT(Page
->Flags
.Consumer
== MC_USER
);
141 InsertTailList(&UserPageListHead
, &Page
->ListEntry
);
142 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
147 MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn
)
149 PLIST_ENTRY NextListEntry
;
150 PHYSICAL_PAGE
* PageDescriptor
;
154 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
155 Page
= MiGetPfnEntry(PreviousPfn
);
157 ASSERT(Page
->Flags
.Type
== MM_PHYSICAL_PAGE_USED
);
158 ASSERT(Page
->Flags
.Consumer
== MC_USER
);
159 NextListEntry
= (PLIST_ENTRY
)Page
->ListEntry
.Flink
;
160 if (NextListEntry
== &UserPageListHead
)
162 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
165 PageDescriptor
= CONTAINING_RECORD(NextListEntry
, PHYSICAL_PAGE
, ListEntry
);
166 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
167 return PageDescriptor
- MmPfnDatabase
;
172 MmRemoveLRUUserPage(PFN_NUMBER Page
)
174 RemoveEntryList(&MiGetPfnEntry(Page
)->ListEntry
);
179 MiFindContiguousPages(IN PFN_NUMBER LowestPfn
,
180 IN PFN_NUMBER HighestPfn
,
181 IN PFN_NUMBER BoundaryPfn
,
182 IN PFN_NUMBER SizeInPages
,
183 IN MEMORY_CACHING_TYPE CacheType
)
185 PFN_NUMBER Page
, PageCount
, LastPage
, Length
, BoundaryMask
;
190 ASSERT(SizeInPages
!= 0);
193 // Convert the boundary PFN into an alignment mask
195 BoundaryMask
= ~(BoundaryPfn
- 1);
198 // Loop all the physical memory blocks
203 // Capture the base page and length of this memory block
205 Page
= MmPhysicalMemoryBlock
->Run
[i
].BasePage
;
206 PageCount
= MmPhysicalMemoryBlock
->Run
[i
].PageCount
;
209 // Check how far this memory block will go
211 LastPage
= Page
+ PageCount
;
214 // Trim it down to only the PFNs we're actually interested in
216 if ((LastPage
- 1) > HighestPfn
) LastPage
= HighestPfn
+ 1;
217 if (Page
< LowestPfn
) Page
= LowestPfn
;
220 // Skip this run if it's empty or fails to contain all the pages we need
222 if (!(PageCount
) || ((Page
+ SizeInPages
) > LastPage
)) continue;
225 // Now scan all the relevant PFNs in this run
228 for (Pfn1
= MiGetPfnEntry(Page
); Page
< LastPage
; Page
++, Pfn1
++)
231 // If this PFN is in use, ignore it
233 if (Pfn1
->Flags
.Type
!= MM_PHYSICAL_PAGE_FREE
) continue;
236 // If we haven't chosen a start PFN yet and the caller specified an
237 // alignment, make sure the page matches the alignment restriction
239 if ((!(Length
) && (BoundaryPfn
)) &&
240 (((Page
^ (Page
+ SizeInPages
- 1)) & BoundaryMask
)))
243 // It does not, so bail out
249 // Increase the number of valid pages, and check if we have enough
251 if (++Length
== SizeInPages
)
254 // It appears we've amassed enough legitimate pages, rollback
256 Pfn1
-= (Length
- 1);
257 Page
-= (Length
- 1);
260 // Acquire the PFN lock
262 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
266 // Things might've changed for us. Is the page still free?
268 if (Pfn1
->Flags
.Type
!= MM_PHYSICAL_PAGE_FREE
) break;
271 // So far so good. Is this the last confirmed valid page?
276 // Sanity check that we didn't go out of bounds
278 ASSERT(i
!= MmPhysicalMemoryBlock
->NumberOfRuns
);
281 // Loop until all PFN entries have been processed
283 EndPfn
= Pfn1
- SizeInPages
+ 1;
287 // If this was an unzeroed page, there are now less
289 if (Pfn1
->Flags
.Zero
== 0) UnzeroedPageCount
--;
292 // One less free page
297 // This PFN is now a used page, set it up
299 RemoveEntryList(&Pfn1
->ListEntry
);
300 Pfn1
->Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
301 Pfn1
->Flags
.Consumer
= MC_NPPOOL
;
302 Pfn1
->ReferenceCount
= 1;
304 Pfn1
->SavedSwapEntry
= 0;
307 // Check if it was already zeroed
309 if (Pfn1
->Flags
.Zero
== 0)
312 // It wasn't, so zero it
314 MiZeroPage(MiGetPfnEntryIndex(Pfn1
));
318 // Check if this is the last PFN, otherwise go on
320 if (Pfn1
== EndPfn
) break;
325 // Mark the first and last PFN so we can find them later
327 Pfn1
->Flags
.StartOfAllocation
= 1;
328 (Pfn1
+ SizeInPages
- 1)->Flags
.EndOfAllocation
= 1;
331 // Now it's safe to let go of the PFN lock
333 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
336 // Quick sanity check that the last PFN is consistent
338 EndPfn
= Pfn1
+ SizeInPages
;
339 ASSERT(EndPfn
== MiGetPfnEntry(Page
+ 1));
342 // Compute the first page, and make sure it's consistent
344 Page
-= SizeInPages
- 1;
345 ASSERT(Pfn1
== MiGetPfnEntry(Page
));
351 // Keep going. The purpose of this loop is to reconfirm that
352 // after acquiring the PFN lock these pages are still usable
359 // If we got here, something changed while we hadn't acquired
360 // the PFN lock yet, so we'll have to restart
362 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
366 } while (++i
!= MmPhysicalMemoryBlock
->NumberOfRuns
);
369 // And if we get here, it means no suitable physical memory runs were found
376 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress
,
377 IN PHYSICAL_ADDRESS HighAddress
,
378 IN PHYSICAL_ADDRESS SkipBytes
,
379 IN SIZE_T TotalBytes
,
380 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
,
384 PFN_NUMBER PageCount
, LowPage
, HighPage
, SkipPages
, PagesFound
= 0, Page
;
385 PPFN_NUMBER MdlPage
, LastMdlPage
;
387 PLIST_ENTRY ListEntry
;
389 INT LookForZeroedPages
;
390 ASSERT (KeGetCurrentIrql() <= APC_LEVEL
);
393 // Convert the low address into a PFN
395 LowPage
= (PFN_NUMBER
)(LowAddress
.QuadPart
>> PAGE_SHIFT
);
398 // Convert, and normalize, the high address into a PFN
400 HighPage
= (PFN_NUMBER
)(HighAddress
.QuadPart
>> PAGE_SHIFT
);
401 if (HighPage
> MmHighestPhysicalPage
) HighPage
= MmHighestPhysicalPage
;
404 // Validate skipbytes and convert them into pages
406 if (BYTE_OFFSET(SkipBytes
.LowPart
)) return NULL
;
407 SkipPages
= (PFN_NUMBER
)(SkipBytes
.QuadPart
>> PAGE_SHIFT
);
410 // Now compute the number of pages the MDL will cover
412 PageCount
= (PFN_NUMBER
)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes
);
416 // Try creating an MDL for these many pages
418 Mdl
= MmCreateMdl(NULL
, NULL
, PageCount
<< PAGE_SHIFT
);
422 // This function is not required to return the amount of pages requested
423 // In fact, it can return as little as 1 page, and callers are supposed
424 // to deal with this scenario. So re-attempt the allocation with less
425 // pages than before, and see if it worked this time.
427 PageCount
-= (PageCount
>> 4);
431 // Wow, not even a single page was around!
433 if (!Mdl
) return NULL
;
436 // This is where the page array starts....
438 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
441 // Lock the PFN database
443 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
446 // Are we looking for any pages, without discriminating?
448 if ((LowPage
== 0) && (HighPage
== MmHighestPhysicalPage
))
451 // Well then, let's go shopping
453 while (PagesFound
< PageCount
)
456 // Do we have zeroed pages?
458 if (!IsListEmpty(&FreeZeroedPageListHead
))
463 ListEntry
= RemoveTailList(&FreeZeroedPageListHead
);
465 else if (!IsListEmpty(&FreeUnzeroedPageListHead
))
468 // Nope, grab an unzeroed page
470 ListEntry
= RemoveTailList(&FreeUnzeroedPageListHead
);
476 // This is not good... hopefully we have at least SOME pages
483 // Get the PFN entry for this page
485 Pfn1
= CONTAINING_RECORD(ListEntry
, PHYSICAL_PAGE
, ListEntry
);
488 // Make sure it's really free
490 ASSERT(Pfn1
->Flags
.Type
== MM_PHYSICAL_PAGE_FREE
);
491 ASSERT(Pfn1
->ReferenceCount
== 0);
494 // Allocate it and mark it
496 Pfn1
->Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
497 Pfn1
->Flags
.Consumer
= MC_NPPOOL
;
498 Pfn1
->Flags
.StartOfAllocation
= 1;
499 Pfn1
->Flags
.EndOfAllocation
= 1;
500 Pfn1
->ReferenceCount
= 1;
502 Pfn1
->SavedSwapEntry
= 0;
505 // Decrease available pages
510 // Save it into the MDL
512 *MdlPage
++ = MiGetPfnEntryIndex(Pfn1
);
519 // You want specific range of pages. We'll do this in two runs
521 for (LookForZeroedPages
= 1; LookForZeroedPages
>= 0; LookForZeroedPages
--)
524 // Scan the range you specified
526 for (Page
= LowPage
; Page
< HighPage
; Page
++)
529 // Get the PFN entry for this page
531 Pfn1
= MiGetPfnEntry(Page
);
535 // Make sure it's free and if this is our first pass, zeroed
537 if (Pfn1
->Flags
.Type
!= MM_PHYSICAL_PAGE_FREE
) continue;
538 if (Pfn1
->Flags
.Zero
!= LookForZeroedPages
) continue;
543 ASSERT(Pfn1
->ReferenceCount
== 0);
546 // Now setup the page and mark it
548 Pfn1
->Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
549 Pfn1
->Flags
.Consumer
= MC_NPPOOL
;
550 Pfn1
->ReferenceCount
= 1;
551 Pfn1
->Flags
.StartOfAllocation
= 1;
552 Pfn1
->Flags
.EndOfAllocation
= 1;
554 Pfn1
->SavedSwapEntry
= 0;
557 // If this page was unzeroed, we've consumed such a page
559 if (!Pfn1
->Flags
.Zero
) UnzeroedPageCount
--;
562 // Decrease available pages
567 // Save this page into the MDL
570 if (++PagesFound
== PageCount
) break;
574 // If the first pass was enough, don't keep going, otherwise, go again
576 if (PagesFound
== PageCount
) break;
581 // Now release the PFN count
583 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
586 // We might've found less pages, but not more ;-)
588 if (PagesFound
!= PageCount
) ASSERT(PagesFound
< PageCount
);
592 // If we didn' tfind any pages at all, fail
594 DPRINT1("NO MDL PAGES!\n");
600 // Write out how many pages we found
602 Mdl
->ByteCount
= (ULONG
)(PagesFound
<< PAGE_SHIFT
);
605 // Terminate the MDL array if there's certain missing pages
607 if (PagesFound
!= PageCount
) *MdlPage
= -1;
610 // Now go back and loop over all the MDL pages
612 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
613 LastMdlPage
= MdlPage
+ PagesFound
;
614 while (MdlPage
< LastMdlPage
)
617 // Check if we've reached the end
620 if (Page
== (PFN_NUMBER
)-1) break;
623 // Get the PFN entry for the page and check if we should zero it out
625 Pfn1
= MiGetPfnEntry(Page
);
627 if (Pfn1
->Flags
.Zero
== 0) MiZeroPage(Page
);
631 // We're done, mark the pages as locked (should we lock them, though???)
634 Mdl
->MdlFlags
|= MDL_PAGES_LOCKED
;
640 MmDumpPfnDatabase(VOID
)
644 PCHAR State
= "????", Consumer
= "Unknown";
646 ULONG Totals
[5] = {0}, FreePages
= 0;
648 KeRaiseIrql(HIGH_LEVEL
, &OldIrql
);
651 // Loop the PFN database
653 for (i
= 0; i
<= MmHighestPhysicalPage
; i
++)
655 Pfn1
= MiGetPfnEntry(i
);
661 switch (Pfn1
->Flags
.Consumer
)
665 Consumer
= "Nonpaged Pool";
670 Consumer
= "Paged Pool";
675 Consumer
= "File System Cache";
680 Consumer
= "Process Working Set";
692 switch (Pfn1
->Flags
.Type
)
694 case MM_PHYSICAL_PAGE_USED
:
697 Totals
[Pfn1
->Flags
.Consumer
]++;
700 case MM_PHYSICAL_PAGE_FREE
:
709 // Pretty-print the page
711 DbgPrint("0x%08p:\t%04s\t%20s\t(%02d.%02d) [%08p])\n",
715 Pfn1
->ReferenceCount
,
720 DbgPrint("Nonpaged Pool: %d pages\t[%d KB]\n", Totals
[MC_NPPOOL
], (Totals
[MC_NPPOOL
] << PAGE_SHIFT
) / 1024);
721 DbgPrint("Paged Pool: %d pages\t[%d KB]\n", Totals
[MC_PPOOL
], (Totals
[MC_PPOOL
] << PAGE_SHIFT
) / 1024);
722 DbgPrint("File System Cache: %d pages\t[%d KB]\n", Totals
[MC_CACHE
], (Totals
[MC_CACHE
] << PAGE_SHIFT
) / 1024);
723 DbgPrint("Process Working Set: %d pages\t[%d KB]\n", Totals
[MC_USER
], (Totals
[MC_USER
] << PAGE_SHIFT
) / 1024);
724 DbgPrint("System: %d pages\t[%d KB]\n", Totals
[MC_SYSTEM
], (Totals
[MC_SYSTEM
] << PAGE_SHIFT
) / 1024);
725 DbgPrint("Free: %d pages\t[%d KB]\n", FreePages
, (FreePages
<< PAGE_SHIFT
) / 1024);
727 KeLowerIrql(OldIrql
);
732 MmInitializePageList(VOID
)
735 PHYSICAL_PAGE UsedPage
;
736 PMEMORY_ALLOCATION_DESCRIPTOR Md
;
737 PLIST_ENTRY NextEntry
;
738 ULONG NrSystemPages
= 0;
740 /* Initialize the page lists */
741 InitializeListHead(&UserPageListHead
);
742 InitializeListHead(&FreeUnzeroedPageListHead
);
743 InitializeListHead(&FreeZeroedPageListHead
);
745 /* This is what a used page looks like */
746 RtlZeroMemory(&UsedPage
, sizeof(UsedPage
));
747 UsedPage
.Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
748 UsedPage
.Flags
.Consumer
= MC_NPPOOL
;
749 UsedPage
.ReferenceCount
= 1;
751 /* Loop the memory descriptors */
752 for (NextEntry
= KeLoaderBlock
->MemoryDescriptorListHead
.Flink
;
753 NextEntry
!= &KeLoaderBlock
->MemoryDescriptorListHead
;
754 NextEntry
= NextEntry
->Flink
)
756 /* Get the descriptor */
757 Md
= CONTAINING_RECORD(NextEntry
,
758 MEMORY_ALLOCATION_DESCRIPTOR
,
761 /* Skip bad memory */
762 if ((Md
->MemoryType
== LoaderFirmwarePermanent
) ||
763 (Md
->MemoryType
== LoaderBBTMemory
) ||
764 (Md
->MemoryType
== LoaderSpecialMemory
) ||
765 (Md
->MemoryType
== LoaderBad
))
768 // We do not build PFN entries for this
772 else if ((Md
->MemoryType
== LoaderFree
) ||
773 (Md
->MemoryType
== LoaderLoadedProgram
) ||
774 (Md
->MemoryType
== LoaderFirmwareTemporary
) ||
775 (Md
->MemoryType
== LoaderOsloaderStack
))
777 /* Loop every page part of the block */
778 for (i
= 0; i
< Md
->PageCount
; i
++)
780 /* Mark it as a free page */
781 MmPfnDatabase
[Md
->BasePage
+ i
].Flags
.Type
= MM_PHYSICAL_PAGE_FREE
;
782 InsertTailList(&FreeUnzeroedPageListHead
,
783 &MmPfnDatabase
[Md
->BasePage
+ i
].ListEntry
);
790 /* Loop every page part of the block */
791 for (i
= 0; i
< Md
->PageCount
; i
++)
793 /* Everything else is used memory */
794 MmPfnDatabase
[Md
->BasePage
+ i
] = UsedPage
;
800 /* Finally handle the pages describing the PFN database themselves */
801 for (i
= MxOldFreeDescriptor
.BasePage
; i
< MxFreeDescriptor
->BasePage
; i
++)
803 /* Ensure this page was not added previously */
804 ASSERT(MmPfnDatabase
[i
].Flags
.Type
== 0);
806 /* Mark it as used kernel memory */
807 MmPfnDatabase
[i
] = UsedPage
;
811 KeInitializeEvent(&ZeroPageThreadEvent
, NotificationEvent
, TRUE
);
812 DPRINT("Pages: %x %x\n", MmAvailablePages
, NrSystemPages
);
813 MmInitializeBalancer(MmAvailablePages
, NrSystemPages
);
818 MmSetRmapListHeadPage(PFN_NUMBER Pfn
, struct _MM_RMAP_ENTRY
* ListHead
)
822 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
823 MiGetPfnEntry(Pfn
)->RmapListHead
= (LONG_PTR
)ListHead
;
824 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
827 struct _MM_RMAP_ENTRY
*
829 MmGetRmapListHeadPage(PFN_NUMBER Pfn
)
832 struct _MM_RMAP_ENTRY
* ListHead
;
834 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
835 ListHead
= (struct _MM_RMAP_ENTRY
*)MiGetPfnEntry(Pfn
)->RmapListHead
;
836 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
843 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn
, SWAPENTRY SwapEntry
)
847 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
848 MiGetPfnEntry(Pfn
)->SavedSwapEntry
= SwapEntry
;
849 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
854 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn
)
859 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
860 SwapEntry
= MiGetPfnEntry(Pfn
)->SavedSwapEntry
;
861 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
868 MmReferencePage(PFN_NUMBER Pfn
)
872 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
874 if (Pfn
== 0 || Pfn
> MmHighestPhysicalPage
)
879 Page
= MiGetPfnEntry(Pfn
);
881 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
883 DPRINT1("Referencing non-used page\n");
884 KeBugCheck(MEMORY_MANAGEMENT
);
887 Page
->ReferenceCount
++;
892 MmGetReferenceCountPage(PFN_NUMBER Pfn
)
898 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
900 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
901 Page
= MiGetPfnEntry(Pfn
);
903 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
905 DPRINT1("Getting reference count for free page\n");
906 KeBugCheck(MEMORY_MANAGEMENT
);
909 RCount
= Page
->ReferenceCount
;
911 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
917 MmIsPageInUse(PFN_NUMBER Pfn
)
920 DPRINT("MmIsPageInUse(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
922 return (MiGetPfnEntry(Pfn
)->Flags
.Type
== MM_PHYSICAL_PAGE_USED
);
927 MmDereferencePage(PFN_NUMBER Pfn
)
931 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
933 Page
= MiGetPfnEntry(Pfn
);
936 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
938 DPRINT1("Dereferencing free page\n");
939 KeBugCheck(MEMORY_MANAGEMENT
);
941 if (Page
->ReferenceCount
== 0)
943 DPRINT1("Derefrencing page with reference count 0\n");
944 KeBugCheck(MEMORY_MANAGEMENT
);
947 Page
->ReferenceCount
--;
948 if (Page
->ReferenceCount
== 0)
951 if (Page
->Flags
.Consumer
== MC_USER
) RemoveEntryList(&Page
->ListEntry
);
952 if (Page
->RmapListHead
!= (LONG_PTR
)NULL
)
954 DPRINT1("Freeing page with rmap entries.\n");
955 KeBugCheck(MEMORY_MANAGEMENT
);
957 if (Page
->LockCount
> 0)
959 DPRINT1("Freeing locked page\n");
960 KeBugCheck(MEMORY_MANAGEMENT
);
962 if (Page
->SavedSwapEntry
!= 0)
964 DPRINT1("Freeing page with swap entry.\n");
965 KeBugCheck(MEMORY_MANAGEMENT
);
967 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
969 DPRINT1("Freeing page with flags %x\n",
971 KeBugCheck(MEMORY_MANAGEMENT
);
973 Page
->Flags
.Type
= MM_PHYSICAL_PAGE_FREE
;
974 Page
->Flags
.Consumer
= MC_MAXIMUM
;
975 InsertTailList(&FreeUnzeroedPageListHead
,
978 if (UnzeroedPageCount
> 8 && 0 == KeReadStateEvent(&ZeroPageThreadEvent
))
980 KeSetEvent(&ZeroPageThreadEvent
, IO_NO_INCREMENT
, FALSE
);
987 MmGetLockCountPage(PFN_NUMBER Pfn
)
990 ULONG CurrentLockCount
;
993 DPRINT("MmGetLockCountPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
995 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
997 Page
= MiGetPfnEntry(Pfn
);
999 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
1001 DPRINT1("Getting lock count for free page\n");
1002 KeBugCheck(MEMORY_MANAGEMENT
);
1005 CurrentLockCount
= Page
->LockCount
;
1006 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
1008 return(CurrentLockCount
);
1013 MmLockPage(PFN_NUMBER Pfn
)
1015 PPHYSICAL_PAGE Page
;
1017 DPRINT("MmLockPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
1019 Page
= MiGetPfnEntry(Pfn
);
1021 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
1023 DPRINT1("Locking free page\n");
1024 KeBugCheck(MEMORY_MANAGEMENT
);
1032 MmUnlockPage(PFN_NUMBER Pfn
)
1034 PPHYSICAL_PAGE Page
;
1036 DPRINT("MmUnlockPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
1038 Page
= MiGetPfnEntry(Pfn
);
1040 if (Page
->Flags
.Type
!= MM_PHYSICAL_PAGE_USED
)
1042 DPRINT1("Unlocking free page\n");
1043 KeBugCheck(MEMORY_MANAGEMENT
);
1051 MmAllocPage(ULONG Consumer
, SWAPENTRY SwapEntry
)
1053 PFN_NUMBER PfnOffset
;
1054 PLIST_ENTRY ListEntry
;
1055 PPHYSICAL_PAGE PageDescriptor
;
1056 BOOLEAN NeedClear
= FALSE
;
1058 DPRINT("MmAllocPage()\n");
1060 if (IsListEmpty(&FreeZeroedPageListHead
))
1062 if (IsListEmpty(&FreeUnzeroedPageListHead
))
1064 /* Check if this allocation is for the PFN DB itself */
1065 if (MmNumberOfPhysicalPages
== 0)
1070 DPRINT1("MmAllocPage(): Out of memory\n");
1073 ListEntry
= RemoveTailList(&FreeUnzeroedPageListHead
);
1074 UnzeroedPageCount
--;
1076 PageDescriptor
= CONTAINING_RECORD(ListEntry
, PHYSICAL_PAGE
, ListEntry
);
1082 ListEntry
= RemoveTailList(&FreeZeroedPageListHead
);
1084 PageDescriptor
= CONTAINING_RECORD(ListEntry
, PHYSICAL_PAGE
, ListEntry
);
1087 if (PageDescriptor
->Flags
.Type
!= MM_PHYSICAL_PAGE_FREE
)
1089 DPRINT1("Got non-free page from freelist\n");
1090 KeBugCheck(MEMORY_MANAGEMENT
);
1092 if (PageDescriptor
->ReferenceCount
!= 0)
1094 DPRINT1("%d\n", PageDescriptor
->ReferenceCount
);
1095 KeBugCheck(MEMORY_MANAGEMENT
);
1097 PageDescriptor
->Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
1098 PageDescriptor
->Flags
.Consumer
= Consumer
;
1099 PageDescriptor
->ReferenceCount
= 1;
1100 PageDescriptor
->LockCount
= 0;
1101 PageDescriptor
->SavedSwapEntry
= SwapEntry
;
1105 PfnOffset
= PageDescriptor
- MmPfnDatabase
;
1106 if ((NeedClear
) && (Consumer
!= MC_SYSTEM
))
1108 MiZeroPage(PfnOffset
);
1115 MiZeroPage(PFN_NUMBER Page
)
1120 Irql
= KeRaiseIrqlToDpcLevel();
1121 TempAddress
= MiMapPageToZeroInHyperSpace(Page
);
1122 if (TempAddress
== NULL
)
1124 return(STATUS_NO_MEMORY
);
1126 memset(TempAddress
, 0, PAGE_SIZE
);
1127 MiUnmapPagesInZeroSpace(TempAddress
, 1);
1129 return(STATUS_SUCCESS
);
1134 MmZeroPageThreadMain(PVOID Ignored
)
1138 PLIST_ENTRY ListEntry
;
1139 PPHYSICAL_PAGE PageDescriptor
;
1143 /* Free initial kernel memory */
1144 //MiFreeInitMemory();
1146 /* Set our priority to 0 */
1147 KeGetCurrentThread()->BasePriority
= 0;
1148 KeSetPriorityThread(KeGetCurrentThread(), 0);
1152 Status
= KeWaitForSingleObject(&ZeroPageThreadEvent
,
1157 if (!NT_SUCCESS(Status
))
1159 DPRINT1("ZeroPageThread: Wait failed\n");
1160 KeBugCheck(MEMORY_MANAGEMENT
);
1163 if (ZeroPageThreadShouldTerminate
)
1165 DPRINT1("ZeroPageThread: Terminating\n");
1166 return STATUS_SUCCESS
;
1169 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1170 while (!IsListEmpty(&FreeUnzeroedPageListHead
))
1172 ListEntry
= RemoveTailList(&FreeUnzeroedPageListHead
);
1173 UnzeroedPageCount
--;
1174 PageDescriptor
= CONTAINING_RECORD(ListEntry
, PHYSICAL_PAGE
, ListEntry
);
1175 /* We set the page to used, because MmCreateVirtualMapping failed with unused pages */
1176 PageDescriptor
->Flags
.Type
= MM_PHYSICAL_PAGE_USED
;
1177 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
1178 Pfn
= PageDescriptor
- MmPfnDatabase
;
1179 Status
= MiZeroPage(Pfn
);
1181 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1182 PageDescriptor
->Flags
.Zero
= 1;
1183 PageDescriptor
->Flags
.Type
= MM_PHYSICAL_PAGE_FREE
;
1184 if (NT_SUCCESS(Status
))
1186 InsertHeadList(&FreeZeroedPageListHead
, ListEntry
);
1191 InsertHeadList(&FreeUnzeroedPageListHead
, ListEntry
);
1192 UnzeroedPageCount
++;
1196 DPRINT("Zeroed %d pages.\n", Count
);
1197 KeResetEvent(&ZeroPageThreadEvent
);
1198 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
1201 return STATUS_SUCCESS
;