2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
11 /* INCLUDES ****************************************************************/
17 #if defined (ALLOC_PRAGMA)
18 #pragma alloc_text(INIT, MmInitializePageList)
21 #define MODULE_INVOLVED_IN_ARM3
22 #include "ARM3/miarm.h"
24 /* GLOBALS ****************************************************************/
28 // ReactOS to NT Physical Page Descriptor Entry Legacy Mapping Definitions
32 #define RmapListHead AweReferenceCount
33 #define PHYSICAL_PAGE MMPFN
34 #define PPHYSICAL_PAGE PMMPFN
36 /* The first array contains ReactOS PFNs, the second contains ARM3 PFNs */
37 PPHYSICAL_PAGE MmPfnDatabase
[2];
39 PFN_NUMBER MmAvailablePages
;
40 PFN_NUMBER MmResidentAvailablePages
;
41 PFN_NUMBER MmResidentAvailableAtInit
;
43 SIZE_T MmTotalCommitLimit
;
44 SIZE_T MmTotalCommittedPages
;
45 SIZE_T MmSharedCommit
;
46 SIZE_T MmDriverCommit
;
47 SIZE_T MmProcessCommit
;
48 SIZE_T MmPagedPoolCommit
;
49 SIZE_T MmPeakCommitment
;
50 SIZE_T MmtotalCommitLimitMaximum
;
52 KEVENT ZeroPageThreadEvent
;
53 static BOOLEAN ZeroPageThreadShouldTerminate
= FALSE
;
54 static RTL_BITMAP MiUserPfnBitMap
;
56 /* FUNCTIONS *************************************************************/
60 MiInitializeUserPfnBitmap(VOID
)
64 /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
65 Bitmap
= ExAllocatePoolWithTag(NonPagedPool
,
66 (((MmHighestPhysicalPage
+ 1) + 31) / 32) * 4,
70 /* Initialize it and clear all the bits to begin with */
71 RtlInitializeBitMap(&MiUserPfnBitMap
,
73 MmHighestPhysicalPage
+ 1);
74 RtlClearAllBits(&MiUserPfnBitMap
);
79 MmGetLRUFirstUserPage(VOID
)
84 /* Find the first user page */
85 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
86 Position
= RtlFindSetBits(&MiUserPfnBitMap
, 1, 0);
87 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
88 if (Position
== 0xFFFFFFFF) return 0;
96 MmInsertLRULastUserPage(PFN_TYPE Pfn
)
100 /* Set the page as a user page */
101 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
102 RtlSetBit(&MiUserPfnBitMap
, Pfn
);
103 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
108 MmGetLRUNextUserPage(PFN_TYPE PreviousPfn
)
113 /* Find the next user page */
114 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
115 Position
= RtlFindSetBits(&MiUserPfnBitMap
, 1, PreviousPfn
+ 1);
116 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
117 if (Position
== 0xFFFFFFFF) return 0;
125 MmRemoveLRUUserPage(PFN_TYPE Page
)
127 /* Unset the page as a user page */
128 RtlClearBit(&MiUserPfnBitMap
, Page
);
133 MiIsPfnFree(IN PMMPFN Pfn1
)
135 /* Must be a free or zero page, with no references, linked */
136 return ((Pfn1
->u3
.e1
.PageLocation
<= StandbyPageList
) &&
139 !(Pfn1
->u3
.e2
.ReferenceCount
));
144 MiIsPfnInUse(IN PMMPFN Pfn1
)
146 /* Standby list or higher, unlinked, and with references */
147 return !MiIsPfnFree(Pfn1
);
152 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress
,
153 IN PHYSICAL_ADDRESS HighAddress
,
154 IN PHYSICAL_ADDRESS SkipBytes
,
155 IN SIZE_T TotalBytes
,
156 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
,
160 PFN_NUMBER PageCount
, LowPage
, HighPage
, SkipPages
, PagesFound
= 0, Page
;
161 PPFN_NUMBER MdlPage
, LastMdlPage
;
164 INT LookForZeroedPages
;
165 ASSERT (KeGetCurrentIrql() <= APC_LEVEL
);
168 // Convert the low address into a PFN
170 LowPage
= (PFN_NUMBER
)(LowAddress
.QuadPart
>> PAGE_SHIFT
);
173 // Convert, and normalize, the high address into a PFN
175 HighPage
= (PFN_NUMBER
)(HighAddress
.QuadPart
>> PAGE_SHIFT
);
176 if (HighPage
> MmHighestPhysicalPage
) HighPage
= MmHighestPhysicalPage
;
179 // Validate skipbytes and convert them into pages
181 if (BYTE_OFFSET(SkipBytes
.LowPart
)) return NULL
;
182 SkipPages
= (PFN_NUMBER
)(SkipBytes
.QuadPart
>> PAGE_SHIFT
);
185 // Now compute the number of pages the MDL will cover
187 PageCount
= (PFN_NUMBER
)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes
);
191 // Try creating an MDL for these many pages
193 Mdl
= MmCreateMdl(NULL
, NULL
, PageCount
<< PAGE_SHIFT
);
197 // This function is not required to return the amount of pages requested
198 // In fact, it can return as little as 1 page, and callers are supposed
199 // to deal with this scenario. So re-attempt the allocation with less
200 // pages than before, and see if it worked this time.
202 PageCount
-= (PageCount
>> 4);
206 // Wow, not even a single page was around!
208 if (!Mdl
) return NULL
;
211 // This is where the page array starts....
213 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
216 // Lock the PFN database
218 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
221 // Are we looking for any pages, without discriminating?
223 if ((LowPage
== 0) && (HighPage
== MmHighestPhysicalPage
))
226 // Well then, let's go shopping
228 while (PagesFound
< PageCount
)
231 // Do we have zeroed pages?
233 if (MmZeroedPageListHead
.Total
)
238 Pfn1
= MiRemoveHeadList(&MmZeroedPageListHead
);
240 else if (MmFreePageListHead
.Total
)
243 // Nope, grab an unzeroed page
245 Pfn1
= MiRemoveHeadList(&MmFreePageListHead
);
250 // This is not good... hopefully we have at least SOME pages
257 // Make sure it's really free
259 ASSERT(MiIsPfnInUse(Pfn1
) == FALSE
);
260 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 0);
263 // Allocate it and mark it
265 Pfn1
->u3
.e1
.StartOfAllocation
= 1;
266 Pfn1
->u3
.e1
.EndOfAllocation
= 1;
267 Pfn1
->u3
.e2
.ReferenceCount
= 1;
270 // Decrease available pages
275 // Save it into the MDL
277 *MdlPage
++ = MiGetPfnEntryIndex(Pfn1
);
284 // You want specific range of pages. We'll do this in two runs
286 for (LookForZeroedPages
= 1; LookForZeroedPages
>= 0; LookForZeroedPages
--)
289 // Scan the range you specified
291 for (Page
= LowPage
; Page
< HighPage
; Page
++)
294 // Get the PFN entry for this page
296 Pfn1
= MiGetPfnEntry(Page
);
300 // Make sure it's free and if this is our first pass, zeroed
302 if (MiIsPfnInUse(Pfn1
)) continue;
303 if ((Pfn1
->u3
.e1
.PageLocation
== ZeroedPageList
) != LookForZeroedPages
) continue;
308 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 0);
311 // Now setup the page and mark it
313 Pfn1
->u3
.e2
.ReferenceCount
= 1;
314 Pfn1
->u3
.e1
.StartOfAllocation
= 1;
315 Pfn1
->u3
.e1
.EndOfAllocation
= 1;
318 // Decrease available pages
323 // Save this page into the MDL
326 if (++PagesFound
== PageCount
) break;
330 // If the first pass was enough, don't keep going, otherwise, go again
332 if (PagesFound
== PageCount
) break;
337 // Now release the PFN count
339 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
342 // We might've found less pages, but not more ;-)
344 if (PagesFound
!= PageCount
) ASSERT(PagesFound
< PageCount
);
348 // If we didn' tfind any pages at all, fail
350 DPRINT1("NO MDL PAGES!\n");
356 // Write out how many pages we found
358 Mdl
->ByteCount
= (ULONG
)(PagesFound
<< PAGE_SHIFT
);
361 // Terminate the MDL array if there's certain missing pages
363 if (PagesFound
!= PageCount
) *MdlPage
= -1;
366 // Now go back and loop over all the MDL pages
368 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
369 LastMdlPage
= MdlPage
+ PagesFound
;
370 while (MdlPage
< LastMdlPage
)
373 // Check if we've reached the end
376 if (Page
== (PFN_NUMBER
)-1) break;
379 // Get the PFN entry for the page and check if we should zero it out
381 Pfn1
= MiGetPfnEntry(Page
);
383 if (Pfn1
->u3
.e1
.PageLocation
!= ZeroedPageList
) MiZeroPage(Page
);
384 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
388 // We're done, mark the pages as locked (should we lock them, though???)
391 Mdl
->MdlFlags
|= MDL_PAGES_LOCKED
;
397 MmDumpPfnDatabase(VOID
)
401 PCHAR State
= "????", Type
= "Unknown";
403 ULONG Totals
[5] = {0}, FreePages
= 0;
405 KeRaiseIrql(HIGH_LEVEL
, &OldIrql
);
408 // Loop the PFN database
410 for (i
= 0; i
<= MmHighestPhysicalPage
; i
++)
412 Pfn1
= MiGetPfnEntry(i
);
418 if (MiIsPfnInUse(Pfn1
))
431 // Pretty-print the page
433 DbgPrint("0x%08p:\t%04s\t%20s\t(%02d) [%08p])\n",
437 Pfn1
->u3
.e2
.ReferenceCount
,
441 DbgPrint("Nonpaged Pool: %d pages\t[%d KB]\n", Totals
[MC_NPPOOL
], (Totals
[MC_NPPOOL
] << PAGE_SHIFT
) / 1024);
442 DbgPrint("Paged Pool: %d pages\t[%d KB]\n", Totals
[MC_PPOOL
], (Totals
[MC_PPOOL
] << PAGE_SHIFT
) / 1024);
443 DbgPrint("File System Cache: %d pages\t[%d KB]\n", Totals
[MC_CACHE
], (Totals
[MC_CACHE
] << PAGE_SHIFT
) / 1024);
444 DbgPrint("Process Working Set: %d pages\t[%d KB]\n", Totals
[MC_USER
], (Totals
[MC_USER
] << PAGE_SHIFT
) / 1024);
445 DbgPrint("System: %d pages\t[%d KB]\n", Totals
[MC_SYSTEM
], (Totals
[MC_SYSTEM
] << PAGE_SHIFT
) / 1024);
446 DbgPrint("Free: %d pages\t[%d KB]\n", FreePages
, (FreePages
<< PAGE_SHIFT
) / 1024);
448 KeLowerIrql(OldIrql
);
453 MmInitializePageList(VOID
)
456 PHYSICAL_PAGE UsedPage
;
457 PMEMORY_ALLOCATION_DESCRIPTOR Md
;
458 PLIST_ENTRY NextEntry
;
459 ULONG NrSystemPages
= 0;
462 /* This is what a used page looks like */
463 RtlZeroMemory(&UsedPage
, sizeof(UsedPage
));
464 UsedPage
.u3
.e1
.PageLocation
= ActiveAndValid
;
465 UsedPage
.u3
.e2
.ReferenceCount
= 1;
467 /* Lock PFN database */
468 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
470 /* Loop the memory descriptors */
471 for (NextEntry
= KeLoaderBlock
->MemoryDescriptorListHead
.Flink
;
472 NextEntry
!= &KeLoaderBlock
->MemoryDescriptorListHead
;
473 NextEntry
= NextEntry
->Flink
)
475 /* Get the descriptor */
476 Md
= CONTAINING_RECORD(NextEntry
,
477 MEMORY_ALLOCATION_DESCRIPTOR
,
480 /* Skip bad memory */
481 if ((Md
->MemoryType
== LoaderFirmwarePermanent
) ||
482 (Md
->MemoryType
== LoaderBBTMemory
) ||
483 (Md
->MemoryType
== LoaderSpecialMemory
) ||
484 (Md
->MemoryType
== LoaderBad
))
487 // We do not build PFN entries for this
491 else if ((Md
->MemoryType
== LoaderFree
) ||
492 (Md
->MemoryType
== LoaderLoadedProgram
) ||
493 (Md
->MemoryType
== LoaderFirmwareTemporary
) ||
494 (Md
->MemoryType
== LoaderOsloaderStack
))
496 /* Loop every page part of the block */
497 for (i
= 0; i
< Md
->PageCount
; i
++)
499 /* Mark it as a free page */
500 MmPfnDatabase
[0][Md
->BasePage
+ i
].u3
.e1
.PageLocation
= FreePageList
;
501 MiInsertInListTail(&MmFreePageListHead
,
502 &MmPfnDatabase
[0][Md
->BasePage
+ i
]);
508 /* Loop every page part of the block */
509 for (i
= 0; i
< Md
->PageCount
; i
++)
511 /* Everything else is used memory */
512 MmPfnDatabase
[0][Md
->BasePage
+ i
] = UsedPage
;
518 /* Finally handle the pages describing the PFN database themselves */
519 for (i
= MxOldFreeDescriptor
.BasePage
; i
< MxFreeDescriptor
->BasePage
; i
++)
521 /* Mark it as used kernel memory */
522 MmPfnDatabase
[0][i
] = UsedPage
;
526 /* Release the PFN database lock */
527 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
529 KeInitializeEvent(&ZeroPageThreadEvent
, NotificationEvent
, TRUE
);
530 DPRINT("Pages: %x %x\n", MmAvailablePages
, NrSystemPages
);
531 MmInitializeBalancer(MmAvailablePages
, NrSystemPages
);
536 MmSetRmapListHeadPage(PFN_TYPE Pfn
, struct _MM_RMAP_ENTRY
* ListHead
)
540 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
541 MiGetPfnEntry(Pfn
)->RmapListHead
= (LONG
)ListHead
;
542 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
545 struct _MM_RMAP_ENTRY
*
547 MmGetRmapListHeadPage(PFN_TYPE Pfn
)
550 struct _MM_RMAP_ENTRY
* ListHead
;
552 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
553 ListHead
= (struct _MM_RMAP_ENTRY
*)MiGetPfnEntry(Pfn
)->RmapListHead
;
554 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
561 MmSetSavedSwapEntryPage(PFN_TYPE Pfn
, SWAPENTRY SwapEntry
)
565 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
566 MiGetPfnEntry(Pfn
)->u1
.WsIndex
= SwapEntry
;
567 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
572 MmGetSavedSwapEntryPage(PFN_TYPE Pfn
)
577 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
578 SwapEntry
= MiGetPfnEntry(Pfn
)->u1
.WsIndex
;
579 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
586 MmReferencePage(PFN_TYPE Pfn
)
590 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
592 if (Pfn
== 0 || Pfn
> MmHighestPhysicalPage
)
597 Page
= MiGetPfnEntry(Pfn
);
600 Page
->u3
.e2
.ReferenceCount
++;
605 MmGetReferenceCountPage(PFN_TYPE Pfn
)
611 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
613 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
614 Page
= MiGetPfnEntry(Pfn
);
617 RCount
= Page
->u3
.e2
.ReferenceCount
;
619 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
625 MmIsPageInUse(PFN_TYPE Pfn
)
627 return MiIsPfnInUse(MiGetPfnEntry(Pfn
));
632 MiSetConsumer(IN PFN_TYPE Pfn
,
635 MiGetPfnEntry(Pfn
)->u3
.e1
.PageLocation
= ActiveAndValid
;
640 MmDereferencePage(PFN_TYPE Pfn
)
644 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
646 Page
= MiGetPfnEntry(Pfn
);
649 Page
->u3
.e2
.ReferenceCount
--;
650 if (Page
->u3
.e2
.ReferenceCount
== 0)
653 Page
->u3
.e1
.PageLocation
= FreePageList
;
654 MiInsertInListTail(&MmFreePageListHead
, Page
);
655 if (MmFreePageListHead
.Total
> 8 && 0 == KeReadStateEvent(&ZeroPageThreadEvent
))
657 KeSetEvent(&ZeroPageThreadEvent
, IO_NO_INCREMENT
, FALSE
);
664 MmAllocPage(ULONG Type
)
667 PPHYSICAL_PAGE PageDescriptor
;
668 BOOLEAN NeedClear
= FALSE
;
670 DPRINT("MmAllocPage()\n");
672 if (MmZeroedPageListHead
.Total
== 0)
674 if (MmFreePageListHead
.Total
== 0)
676 /* Check if this allocation is for the PFN DB itself */
677 if (MmNumberOfPhysicalPages
== 0)
682 DPRINT1("MmAllocPage(): Out of memory\n");
685 PageDescriptor
= MiRemoveHeadList(&MmFreePageListHead
);
691 PageDescriptor
= MiRemoveHeadList(&MmZeroedPageListHead
);
694 PageDescriptor
->u3
.e2
.ReferenceCount
= 1;
698 PfnOffset
= PageDescriptor
- MmPfnDatabase
[0];
699 if ((NeedClear
) && (Type
!= MC_SYSTEM
))
701 MiZeroPage(PfnOffset
);
704 PageDescriptor
->u3
.e1
.PageLocation
= ActiveAndValid
;
710 MiZeroPage(PFN_TYPE Page
)
715 Irql
= KeRaiseIrqlToDpcLevel();
716 TempAddress
= MiMapPageToZeroInHyperSpace(Page
);
717 if (TempAddress
== NULL
)
719 return(STATUS_NO_MEMORY
);
721 memset(TempAddress
, 0, PAGE_SIZE
);
722 MiUnmapPagesInZeroSpace(TempAddress
, 1);
724 return(STATUS_SUCCESS
);
729 MmZeroPageThreadMain(PVOID Ignored
)
733 PPHYSICAL_PAGE PageDescriptor
;
737 /* Free initial kernel memory */
738 //MiFreeInitMemory();
740 /* Set our priority to 0 */
741 KeGetCurrentThread()->BasePriority
= 0;
742 KeSetPriorityThread(KeGetCurrentThread(), 0);
746 Status
= KeWaitForSingleObject(&ZeroPageThreadEvent
,
752 if (ZeroPageThreadShouldTerminate
)
754 DPRINT1("ZeroPageThread: Terminating\n");
755 return STATUS_SUCCESS
;
758 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
759 while (MmFreePageListHead
.Total
)
761 PageDescriptor
= MiRemoveHeadList(&MmFreePageListHead
);
762 /* We set the page to used, because MmCreateVirtualMapping failed with unused pages */
763 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
764 Pfn
= PageDescriptor
- MmPfnDatabase
[0];
765 Status
= MiZeroPage(Pfn
);
767 oldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
768 if (NT_SUCCESS(Status
))
770 MiInsertZeroListAtBack(Pfn
);
775 MiInsertInListTail(&MmFreePageListHead
, PageDescriptor
);
776 PageDescriptor
->u3
.e1
.PageLocation
= FreePageList
;
780 DPRINT("Zeroed %d pages.\n", Count
);
781 KeResetEvent(&ZeroPageThreadEvent
);
782 KeReleaseQueuedSpinLock(LockQueuePfnLock
, oldIrql
);
785 return STATUS_SUCCESS
;