Also, implement flushing mapped sections to disk on shutdown.
#include <internal/arch/mm.h>
/* TYPES *********************************************************************/
-
-#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT))
-#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001)
-#define MM_IS_WAIT_PTE(E) \
- (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
-#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT))
-#define SWAPENTRY_FROM_SSE(E) ((E) >> 1)
-#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1)
-#define DIRTY_SSE(E) ((E) | 2)
-#define CLEAN_SSE(E) ((E) & ~2)
-#define IS_DIRTY_SSE(E) ((E) & 2)
-#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000)
-#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2)
-#define MAX_SHARE_COUNT 0x3FF
-#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2)))
-
#define MM_SEGMENT_FINALIZE (0x40000000)
#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff)
NTAPI
MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment);
-NTSTATUS
-NTAPI
-_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
- PLARGE_INTEGER Offset,
- ULONG_PTR Entry,
- const char *file,
- int line);
-
-ULONG_PTR
-NTAPI
-_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
- PLARGE_INTEGER Offset,
- const char *file,
- int line);
-
-#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
-
-#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
-
typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)(
PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset);
MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment,
FREE_SECTION_PAGE_FUN FreePage);
-/* Yields a lock */
-PMM_SECTION_SEGMENT
-NTAPI
-MmGetSectionAssociation(PFN_NUMBER Page,
- PLARGE_INTEGER Offset);
-
NTSTATUS
NTAPI
MmSetSectionAssociation(PFN_NUMBER Page,
PEPROCESS Process,
PVOID Address);
-VOID
-NTAPI
-_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
- const char *file,
- int line);
-
-#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
-
-VOID
-NTAPI
-_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
- const char *file,
- int line);
-
-#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
-
VOID
MmFreeCacheSectionPage(PVOID Context,
MEMORY_AREA* MemoryArea,
// HACK until WS lists are supported
MMWSLE Wsle;
+ struct _MMPFN* NextLRU;
+ struct _MMPFN* PreviousLRU;
} MMPFN, *PMMPFN;
extern PMMPFN MmPfnDatabase;
NTAPI
MmPageOutPhysicalAddress(PFN_NUMBER Page);
+PMM_SECTION_SEGMENT
+NTAPI
+MmGetSectionAssociation(PFN_NUMBER Page,
+ PLARGE_INTEGER Offset);
+
/* freelist.c **********************************************************/
FORCEINLINE
PFN_NUMBER
NTAPI
-MmGetLRUNextUserPage(PFN_NUMBER PreviousPage);
+MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast);
PFN_NUMBER
NTAPI
MmGetLRUFirstUserPage(VOID);
-VOID
-NTAPI
-MmInsertLRULastUserPage(PFN_NUMBER Page);
-
-VOID
-NTAPI
-MmRemoveLRUUserPage(PFN_NUMBER Page);
-
VOID
NTAPI
MmDumpArmPfnDatabase(
/* section.c *****************************************************************/
+#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT))
+#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001)
+#define MM_IS_WAIT_PTE(E) \
+ (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
+#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT))
+#define SWAPENTRY_FROM_SSE(E) ((E) >> 1)
+#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1)
+#define DIRTY_SSE(E) ((E) | 2)
+#define CLEAN_SSE(E) ((E) & ~2)
+#define IS_DIRTY_SSE(E) ((E) & 2)
+#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000)
+#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2)
+#define MAX_SHARE_COUNT 0x3FF
+#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2)))
+
+VOID
+NTAPI
+_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
+ const char *file,
+ int line);
+
+#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
+
+VOID
+NTAPI
+_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
+ const char *file,
+ int line);
+
+#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
+
VOID
NTAPI
MmGetImageInformation(
_In_ PVOID Section,
_Inout_ PLARGE_INTEGER NewSize);
+/* sptab.c *******************************************************************/
+
+NTSTATUS
+NTAPI
+_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
+ PLARGE_INTEGER Offset,
+ ULONG_PTR Entry,
+ const char *file,
+ int line);
+
+ULONG_PTR
+NTAPI
+_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
+ PLARGE_INTEGER Offset,
+ const char *file,
+ int line);
+
+#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
+
+#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
+
/* sysldr.c ******************************************************************/
VOID
KeBugCheck(MEMORY_MANAGEMENT);
}
- if (MmGetReferenceCountPage(Page) == 1)
- {
- if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
- (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
- }
+ (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
MmDereferencePage(Page);
MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
{
PFN_NUMBER CurrentPage;
- PFN_NUMBER NextPage;
NTSTATUS Status;
(*NrFreedPages) = 0;
(*NrFreedPages)++;
}
- NextPage = MmGetLRUNextUserPage(CurrentPage);
- if (NextPage <= CurrentPage)
- {
- /* We wrapped around, so we're done */
- break;
- }
- CurrentPage = NextPage;
+ CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
+ }
+
+ if (CurrentPage)
+ {
+ KIRQL OldIrql = MiAcquirePfnLock();
+ MmDereferencePage(CurrentPage);
+ MiReleasePfnLock(OldIrql);
}
return STATUS_SUCCESS;
/*
* Allocate always memory for the non paged pool and for the pager thread.
*/
- if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */)
+ if (Consumer == MC_SYSTEM)
{
Page = MmAllocPage(Consumer);
if (Page == 0)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
- if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)
MmRebalanceMemoryConsumers();
KeBugCheck(NO_PAGES_AVAILABLE);
}
- if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
- if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)
SIZE_T MmPeakCommitment;
SIZE_T MmtotalCommitLimitMaximum;
-static RTL_BITMAP MiUserPfnBitMap;
+PMMPFN FirstUserLRUPfn;
+PMMPFN LastUserLRUPfn;
/* FUNCTIONS *************************************************************/
-VOID
-NTAPI
-MiInitializeUserPfnBitmap(VOID)
-{
- PVOID Bitmap;
-
- /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
- Bitmap = ExAllocatePoolWithTag(NonPagedPool,
- (((MmHighestPhysicalPage + 1) + 31) / 32) * 4,
- TAG_MM);
- ASSERT(Bitmap);
-
- /* Initialize it and clear all the bits to begin with */
- RtlInitializeBitMap(&MiUserPfnBitMap,
- Bitmap,
- (ULONG)MmHighestPhysicalPage + 1);
- RtlClearAllBits(&MiUserPfnBitMap);
-}
-
PFN_NUMBER
NTAPI
MmGetLRUFirstUserPage(VOID)
{
- ULONG Position;
+ PFN_NUMBER Page;
KIRQL OldIrql;
/* Find the first user page */
OldIrql = MiAcquirePfnLock();
- Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0);
+
+ if (FirstUserLRUPfn == NULL)
+ {
+ MiReleasePfnLock(OldIrql);
+ return 0;
+ }
+
+ Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
+ MmReferencePage(Page);
+
MiReleasePfnLock(OldIrql);
- if (Position == 0xFFFFFFFF) return 0;
- /* Return it */
- ASSERT(Position != 0);
- ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
- return Position;
+ return Page;
}
+static
VOID
-NTAPI
-MmInsertLRULastUserPage(PFN_NUMBER Pfn)
+MmInsertLRULastUserPage(PFN_NUMBER Page)
{
- KIRQL OldIrql;
+ MI_ASSERT_PFN_LOCK_HELD();
- /* Set the page as a user page */
- ASSERT(Pfn != 0);
- ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn));
- ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn));
- OldIrql = MiAcquirePfnLock();
- RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn);
- MiReleasePfnLock(OldIrql);
-}
+ PMMPFN Pfn = MiGetPfnEntry(Page);
-PFN_NUMBER
-NTAPI
-MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn)
-{
- ULONG Position;
- KIRQL OldIrql;
+ if (FirstUserLRUPfn == NULL)
+ FirstUserLRUPfn = Pfn;
- /* Find the next user page */
- OldIrql = MiAcquirePfnLock();
- Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1);
- MiReleasePfnLock(OldIrql);
- if (Position == 0xFFFFFFFF) return 0;
+ Pfn->PreviousLRU = LastUserLRUPfn;
- /* Return it */
- ASSERT(Position != 0);
- ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
- return Position;
+ if (LastUserLRUPfn != NULL)
+ LastUserLRUPfn->NextLRU = Pfn;
+ LastUserLRUPfn = Pfn;
}
+static
VOID
-NTAPI
MmRemoveLRUUserPage(PFN_NUMBER Page)
{
- KIRQL OldIrql;
+ MI_ASSERT_PFN_LOCK_HELD();
/* Unset the page as a user page */
ASSERT(Page != 0);
- ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page));
- ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page));
+
+ PMMPFN Pfn = MiGetPfnEntry(Page);
+
+ ASSERT_IS_ROS_PFN(Pfn);
+
+ if (Pfn->PreviousLRU)
+ {
+ ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
+ Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
+ }
+ else
+ {
+ ASSERT(FirstUserLRUPfn == Pfn);
+ FirstUserLRUPfn = Pfn->NextLRU;
+ }
+
+ if (Pfn->NextLRU)
+ {
+ ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
+ Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
+ }
+ else
+ {
+ ASSERT(Pfn == LastUserLRUPfn);
+ LastUserLRUPfn = Pfn->PreviousLRU;
+ }
+
+ Pfn->PreviousLRU = Pfn->NextLRU = NULL;
+}
+
+PFN_NUMBER
+NTAPI
+MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
+{
+ PFN_NUMBER Page = 0;
+ KIRQL OldIrql;
+
+ /* Find the next user page */
OldIrql = MiAcquirePfnLock();
- RtlClearBit(&MiUserPfnBitMap, (ULONG)Page);
+
+ PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
+ PMMPFN NextPfn = PreviousPfn->NextLRU;
+
+ /*
+ * Move this one at the end of the list.
+ * It may be freed by MmDereferencePage below.
+ * If it's not, then it means it is still hanging in some process address space.
+ * This avoids paging-out e.g. ntdll early just because it's mapped first time.
+ */
+ if (MoveToLast)
+ {
+ MmRemoveLRUUserPage(PreviousPage);
+ MmInsertLRULastUserPage(PreviousPage);
+ }
+
+ if (NextPfn)
+ {
+ Page = MiGetPfnEntryIndex(NextPfn);
+ MmReferencePage(Page);
+ }
+
+ MmDereferencePage(PreviousPage);
+
MiReleasePfnLock(OldIrql);
+
+ return Page;
}
BOOLEAN
Pfn1->u3.e2.ReferenceCount--;
if (Pfn1->u3.e2.ReferenceCount == 0)
{
+ /* Apply LRU hack */
+ if (Pfn1->u4.MustBeCached)
+ {
+ MmRemoveLRUUserPage(Pfn);
+ Pfn1->u4.MustBeCached = 0;
+ }
+
/* Mark the page temporarily as valid, we're going to make it free soon */
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u1.SwapEntry = 0;
Pfn1->RmapListHead = NULL;
+ Pfn1->NextLRU = NULL;
+ Pfn1->PreviousLRU = NULL;
+
+ if (Type == MC_USER)
+ {
+ Pfn1->u4.MustBeCached = 1; /* HACK again */
+ MmInsertLRULastUserPage(PfnOffset);
+ }
+
MiReleasePfnLock(OldIrql);
return PfnOffset;
}
/* GLOBALS *******************************************************************/
-VOID NTAPI MiInitializeUserPfnBitmap(VOID);
-
BOOLEAN Mm64BitPhysicalAddress = FALSE;
ULONG MmReadClusterSize;
//
MiDbgDumpAddressSpace();
MmInitGlobalKernelPageDirectory();
- MiInitializeUserPfnBitmap();
MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
MmInitializeRmapList();
MmInitSectionImplementation();
MiShutdownSystem(VOID)
{
ULONG i;
+ PFN_NUMBER Page;
+ BOOLEAN Dirty;
+
+ /* Loop through all the pages owned by the legacy Mm and page them out, if needed. */
+ /* We do it twice, since flushing can cause the FS to dirtify new pages */
+ do
+ {
+ Dirty = FALSE;
+
+ Page = MmGetLRUFirstUserPage();
+ while (Page)
+ {
+ LARGE_INTEGER SegmentOffset;
+ PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page, &SegmentOffset);
+
+ if (Segment)
+ {
+ if ((*Segment->Flags) & MM_DATAFILE_SEGMENT)
+ {
+ MmLockSectionSegment(Segment);
+
+ ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
+
+ if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry))
+ {
+ Dirty = TRUE;
+ MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
+ }
+
+ MmUnlockSectionSegment(Segment);
+ }
+
+ MmDereferenceSegment(Segment);
+ }
+
+ Page = MmGetLRUNextUserPage(Page, FALSE);
+ }
+ } while (Dirty);
/* Loop through all the paging files */
for (i = 0; i < MmNumberOfPagingFiles; i++)
projects / reactos.git / commitdiff