2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
11 /* INCLUDES ****************************************************************/
17 #define MODULE_INVOLVED_IN_ARM3
18 #include "ARM3/miarm.h"
20 #define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
22 /* GLOBALS ****************************************************************/
26 PFN_NUMBER MmAvailablePages
;
27 PFN_NUMBER MmResidentAvailablePages
;
28 PFN_NUMBER MmResidentAvailableAtInit
;
30 SIZE_T MmTotalCommittedPages
;
31 SIZE_T MmSharedCommit
;
32 SIZE_T MmDriverCommit
;
33 SIZE_T MmProcessCommit
;
34 SIZE_T MmPagedPoolCommit
;
35 SIZE_T MmPeakCommitment
;
36 SIZE_T MmtotalCommitLimitMaximum
;
38 static RTL_BITMAP MiUserPfnBitMap
;
40 /* FUNCTIONS *************************************************************/
44 MiInitializeUserPfnBitmap(VOID
)
48 /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
49 Bitmap
= ExAllocatePoolWithTag(NonPagedPool
,
50 (((MmHighestPhysicalPage
+ 1) + 31) / 32) * 4,
54 /* Initialize it and clear all the bits to begin with */
55 RtlInitializeBitMap(&MiUserPfnBitMap
,
57 (ULONG
)MmHighestPhysicalPage
+ 1);
58 RtlClearAllBits(&MiUserPfnBitMap
);
63 MmGetLRUFirstUserPage(VOID
)
68 /* Find the first user page */
69 OldIrql
= MiAcquirePfnLock();
70 Position
= RtlFindSetBits(&MiUserPfnBitMap
, 1, 0);
71 MiReleasePfnLock(OldIrql
);
72 if (Position
== 0xFFFFFFFF) return 0;
75 ASSERT(Position
!= 0);
76 ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position
));
82 MmInsertLRULastUserPage(PFN_NUMBER Pfn
)
86 /* Set the page as a user page */
88 ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn
));
89 ASSERT(!RtlCheckBit(&MiUserPfnBitMap
, (ULONG
)Pfn
));
90 OldIrql
= MiAcquirePfnLock();
91 RtlSetBit(&MiUserPfnBitMap
, (ULONG
)Pfn
);
92 MiReleasePfnLock(OldIrql
);
97 MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn
)
102 /* Find the next user page */
103 OldIrql
= MiAcquirePfnLock();
104 Position
= RtlFindSetBits(&MiUserPfnBitMap
, 1, (ULONG
)PreviousPfn
+ 1);
105 MiReleasePfnLock(OldIrql
);
106 if (Position
== 0xFFFFFFFF) return 0;
109 ASSERT(Position
!= 0);
110 ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position
));
116 MmRemoveLRUUserPage(PFN_NUMBER Page
)
120 /* Unset the page as a user page */
122 ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page
));
123 ASSERT(RtlCheckBit(&MiUserPfnBitMap
, (ULONG
)Page
));
124 OldIrql
= MiAcquirePfnLock();
125 RtlClearBit(&MiUserPfnBitMap
, (ULONG
)Page
);
126 MiReleasePfnLock(OldIrql
);
131 MiIsPfnFree(IN PMMPFN Pfn1
)
133 /* Must be a free or zero page, with no references, linked */
134 return ((Pfn1
->u3
.e1
.PageLocation
<= StandbyPageList
) &&
137 !(Pfn1
->u3
.e2
.ReferenceCount
));
142 MiIsPfnInUse(IN PMMPFN Pfn1
)
144 /* Standby list or higher, unlinked, and with references */
145 return !MiIsPfnFree(Pfn1
);
150 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress
,
151 IN PHYSICAL_ADDRESS HighAddress
,
152 IN PHYSICAL_ADDRESS SkipBytes
,
153 IN SIZE_T TotalBytes
,
154 IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute
,
158 PFN_NUMBER PageCount
, LowPage
, HighPage
, SkipPages
, PagesFound
= 0, Page
;
159 PPFN_NUMBER MdlPage
, LastMdlPage
;
162 INT LookForZeroedPages
;
163 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
164 DPRINT1("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress
, HighAddress
, SkipBytes
, TotalBytes
, CacheAttribute
, MdlFlags
);
167 // Convert the low address into a PFN
169 LowPage
= (PFN_NUMBER
)(LowAddress
.QuadPart
>> PAGE_SHIFT
);
172 // Convert, and normalize, the high address into a PFN
174 HighPage
= (PFN_NUMBER
)(HighAddress
.QuadPart
>> PAGE_SHIFT
);
175 if (HighPage
> MmHighestPhysicalPage
) HighPage
= MmHighestPhysicalPage
;
178 // Validate skipbytes and convert them into pages
180 if (BYTE_OFFSET(SkipBytes
.LowPart
)) return NULL
;
181 SkipPages
= (PFN_NUMBER
)(SkipBytes
.QuadPart
>> PAGE_SHIFT
);
183 /* This isn't supported at all */
184 if (SkipPages
) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
187 // Now compute the number of pages the MDL will cover
189 PageCount
= (PFN_NUMBER
)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes
);
193 // Try creating an MDL for these many pages
195 Mdl
= MmCreateMdl(NULL
, NULL
, PageCount
<< PAGE_SHIFT
);
199 // This function is not required to return the amount of pages requested
200 // In fact, it can return as little as 1 page, and callers are supposed
201 // to deal with this scenario. So re-attempt the allocation with less
202 // pages than before, and see if it worked this time.
204 PageCount
-= (PageCount
>> 4);
208 // Wow, not even a single page was around!
210 if (!Mdl
) return NULL
;
213 // This is where the page array starts....
215 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
218 // Lock the PFN database
220 OldIrql
= MiAcquirePfnLock();
223 // Are we looking for any pages, without discriminating?
225 if ((LowPage
== 0) && (HighPage
== MmHighestPhysicalPage
))
228 // Well then, let's go shopping
230 while (PagesFound
< PageCount
)
233 MI_SET_USAGE(MI_USAGE_MDL
);
234 MI_SET_PROCESS2("Kernel");
236 /* FIXME: This check should be smarter */
238 if (MmAvailablePages
!= 0)
239 Page
= MiRemoveAnyPage(0);
243 /* This is not good... hopefully we have at least SOME pages */
248 /* Grab the page entry for it */
249 Pfn1
= MiGetPfnEntry(Page
);
252 // Make sure it's really free
254 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 0);
256 /* Now setup the page and mark it */
257 Pfn1
->u3
.e2
.ReferenceCount
= 1;
258 Pfn1
->u2
.ShareCount
= 1;
259 MI_SET_PFN_DELETED(Pfn1
);
260 Pfn1
->u4
.PteFrame
= 0x1FFEDCB;
261 Pfn1
->u3
.e1
.StartOfAllocation
= 1;
262 Pfn1
->u3
.e1
.EndOfAllocation
= 1;
263 Pfn1
->u4
.VerifierAllocation
= 0;
266 // Save it into the MDL
268 *MdlPage
++ = MiGetPfnEntryIndex(Pfn1
);
275 // You want specific range of pages. We'll do this in two runs
277 for (LookForZeroedPages
= 1; LookForZeroedPages
>= 0; LookForZeroedPages
--)
280 // Scan the range you specified
282 for (Page
= LowPage
; Page
< HighPage
; Page
++)
285 // Get the PFN entry for this page
287 Pfn1
= MiGetPfnEntry(Page
);
291 // Make sure it's free and if this is our first pass, zeroed
293 if (MiIsPfnInUse(Pfn1
)) continue;
294 if ((Pfn1
->u3
.e1
.PageLocation
== ZeroedPageList
) != LookForZeroedPages
) continue;
296 /* Remove the page from the free or zero list */
297 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 0);
298 MI_SET_USAGE(MI_USAGE_MDL
);
299 MI_SET_PROCESS2("Kernel");
300 MiUnlinkFreeOrZeroedPage(Pfn1
);
305 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
== 0);
308 // Now setup the page and mark it
310 Pfn1
->u3
.e2
.ReferenceCount
= 1;
311 Pfn1
->u2
.ShareCount
= 1;
312 MI_SET_PFN_DELETED(Pfn1
);
313 Pfn1
->u4
.PteFrame
= 0x1FFEDCB;
314 Pfn1
->u3
.e1
.StartOfAllocation
= 1;
315 Pfn1
->u3
.e1
.EndOfAllocation
= 1;
316 Pfn1
->u4
.VerifierAllocation
= 0;
319 // Save this page into the MDL
322 if (++PagesFound
== PageCount
) break;
326 // If the first pass was enough, don't keep going, otherwise, go again
328 if (PagesFound
== PageCount
) break;
333 // Now release the PFN count
335 MiReleasePfnLock(OldIrql
);
338 // We might've found less pages, but not more ;-)
340 if (PagesFound
!= PageCount
) ASSERT(PagesFound
< PageCount
);
344 // If we didn' tfind any pages at all, fail
346 DPRINT1("NO MDL PAGES!\n");
347 ExFreePoolWithTag(Mdl
, TAG_MDL
);
352 // Write out how many pages we found
354 Mdl
->ByteCount
= (ULONG
)(PagesFound
<< PAGE_SHIFT
);
357 // Terminate the MDL array if there's certain missing pages
359 if (PagesFound
!= PageCount
) *MdlPage
= LIST_HEAD
;
362 // Now go back and loop over all the MDL pages
364 MdlPage
= (PPFN_NUMBER
)(Mdl
+ 1);
365 LastMdlPage
= MdlPage
+ PagesFound
;
366 while (MdlPage
< LastMdlPage
)
369 // Check if we've reached the end
372 if (Page
== LIST_HEAD
) break;
375 // Get the PFN entry for the page and check if we should zero it out
377 Pfn1
= MiGetPfnEntry(Page
);
379 if (Pfn1
->u3
.e1
.PageLocation
!= ZeroedPageList
) MiZeroPhysicalPage(Page
);
380 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
384 // We're done, mark the pages as locked
387 Mdl
->MdlFlags
|= MDL_PAGES_LOCKED
;
393 MmSetRmapListHeadPage(PFN_NUMBER Pfn
, PMM_RMAP_ENTRY ListHead
)
398 oldIrql
= MiAcquirePfnLock();
399 Pfn1
= MiGetPfnEntry(Pfn
);
401 ASSERT_IS_ROS_PFN(Pfn1
);
405 /* Should not be trying to insert an RMAP for a non-active page */
406 ASSERT(MiIsPfnInUse(Pfn1
) == TRUE
);
408 /* Set the list head address */
409 Pfn1
->RmapListHead
= ListHead
;
413 /* ReactOS semantics dictate the page is STILL active right now */
414 ASSERT(MiIsPfnInUse(Pfn1
) == TRUE
);
416 /* In this case, the RMAP is actually being removed, so clear field */
417 Pfn1
->RmapListHead
= NULL
;
419 /* ReactOS semantics will now release the page, which will make it free and enter a colored list */
422 MiReleasePfnLock(oldIrql
);
427 MmGetRmapListHeadPage(PFN_NUMBER Pfn
)
430 PMM_RMAP_ENTRY ListHead
;
433 /* Lock PFN database */
434 oldIrql
= MiAcquirePfnLock();
437 Pfn1
= MiGetPfnEntry(Pfn
);
439 ASSERT_IS_ROS_PFN(Pfn1
);
441 /* Get the list head */
442 ListHead
= Pfn1
->RmapListHead
;
444 /* Should not have an RMAP for a non-active page */
445 ASSERT(MiIsPfnInUse(Pfn1
) == TRUE
);
447 /* Release PFN database and return rmap list head */
448 MiReleasePfnLock(oldIrql
);
454 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn
, SWAPENTRY SwapEntry
)
459 Pfn1
= MiGetPfnEntry(Pfn
);
461 ASSERT_IS_ROS_PFN(Pfn1
);
463 oldIrql
= MiAcquirePfnLock();
464 Pfn1
->u1
.SwapEntry
= SwapEntry
;
465 MiReleasePfnLock(oldIrql
);
470 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn
)
476 Pfn1
= MiGetPfnEntry(Pfn
);
478 ASSERT_IS_ROS_PFN(Pfn1
);
480 oldIrql
= MiAcquirePfnLock();
481 SwapEntry
= Pfn1
->u1
.SwapEntry
;
482 MiReleasePfnLock(oldIrql
);
489 MmReferencePage(PFN_NUMBER Pfn
)
493 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
495 MI_ASSERT_PFN_LOCK_HELD();
497 ASSERT(Pfn
<= MmHighestPhysicalPage
);
499 Pfn1
= MiGetPfnEntry(Pfn
);
501 ASSERT_IS_ROS_PFN(Pfn1
);
503 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
504 Pfn1
->u3
.e2
.ReferenceCount
++;
509 MmGetReferenceCountPage(PFN_NUMBER Pfn
)
515 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
517 oldIrql
= MiAcquirePfnLock();
518 Pfn1
= MiGetPfnEntry(Pfn
);
520 ASSERT_IS_ROS_PFN(Pfn1
);
522 RCount
= Pfn1
->u3
.e2
.ReferenceCount
;
524 MiReleasePfnLock(oldIrql
);
530 MmIsPageInUse(PFN_NUMBER Pfn
)
532 return MiIsPfnInUse(MiGetPfnEntry(Pfn
));
537 MmDereferencePage(PFN_NUMBER Pfn
)
541 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn
<< PAGE_SHIFT
);
543 OldIrql
= MiAcquirePfnLock();
545 Pfn1
= MiGetPfnEntry(Pfn
);
547 ASSERT_IS_ROS_PFN(Pfn1
);
549 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
550 Pfn1
->u3
.e2
.ReferenceCount
--;
551 if (Pfn1
->u3
.e2
.ReferenceCount
== 0)
553 /* Mark the page temporarily as valid, we're going to make it free soon */
554 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
556 /* It's not a ROS PFN anymore */
557 Pfn1
->u4
.AweAllocation
= FALSE
;
559 /* Bring it back into the free list */
560 DPRINT("Legacy free: %lx\n", Pfn
);
561 MiInsertPageInFreeList(Pfn
);
564 MiReleasePfnLock(OldIrql
);
569 MmAllocPage(ULONG Type
)
571 PFN_NUMBER PfnOffset
;
575 OldIrql
= MiAcquirePfnLock();
577 PfnOffset
= MiRemoveZeroPage(MI_GET_NEXT_COLOR());
580 KeBugCheck(NO_PAGES_AVAILABLE
);
583 DPRINT("Legacy allocate: %lx\n", PfnOffset
);
584 Pfn1
= MiGetPfnEntry(PfnOffset
);
585 Pfn1
->u3
.e2
.ReferenceCount
= 1;
586 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
588 /* This marks the PFN as a ReactOS PFN */
589 Pfn1
->u4
.AweAllocation
= TRUE
;
591 /* Allocate the extra ReactOS Data and zero it out */
592 Pfn1
->u1
.SwapEntry
= 0;
593 Pfn1
->RmapListHead
= NULL
;
595 MiReleasePfnLock(OldIrql
);