Hopefully fail to break anything in the process of syncing with trunk (r47786)

[reactos.git] / ntoskrnl / mm / balance.c

diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c
index 03608ea..cc7b59d 100644 (file)
--- a/ntoskrnl/mm/balance.c
+++ b/ntoskrnl/mm/balance.c
@@ -11,7 +11,7 @@
 
 #include <ntoskrnl.h>
 #define NDEBUG
-#include <internal/debug.h>
+#include <debug.h>
 
 #if defined (ALLOC_PRAGMA)
 #pragma alloc_text(INIT, MmInitializeBalancer)
@@ -49,10 +49,10 @@ static LONG MiBalancerWork = 0;
 
 VOID MmPrintMemoryStatistic(VOID)
 {
-   DbgPrint("MC_CACHE %d, MC_USER %d, MC_PPOOL %d, MC_NPPOOL %d, MmStats.NrFreePages %d\n",
+   DbgPrint("MC_CACHE %d, MC_USER %d, MC_PPOOL %d, MC_NPPOOL %d, MmAvailablePages %d\n",
             MiMemoryConsumers[MC_CACHE].PagesUsed, MiMemoryConsumers[MC_USER].PagesUsed,
             MiMemoryConsumers[MC_PPOOL].PagesUsed, MiMemoryConsumers[MC_NPPOOL].PagesUsed,
-            MmStats.NrFreePages);
+            MmAvailablePages);
 }
 
 VOID
@@ -68,12 +68,25 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
 
    /* Set up targets. */
    MiMinimumAvailablePages = 64;
-   MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 2;
+    if ((NrAvailablePages + NrSystemPages) >= 8192)
+    {
+        MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;   
+    }
+    else if ((NrAvailablePages + NrSystemPages) >= 4096)
+    {
+        MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
+    }
+    else
+    {
+        MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;        
+    }
    MiMemoryConsumers[MC_USER].PagesTarget =
       NrAvailablePages - MiMinimumAvailablePages;
    MiMemoryConsumers[MC_PPOOL].PagesTarget = NrAvailablePages / 2;
    MiMemoryConsumers[MC_NPPOOL].PagesTarget = 0xFFFFFFFF;
    MiMemoryConsumers[MC_NPPOOL].PagesUsed = NrSystemPages;
+   MiMemoryConsumers[MC_SYSTEM].PagesTarget = 0xFFFFFFFF;
+   MiMemoryConsumers[MC_SYSTEM].PagesUsed = 0;
 }
 
 VOID
@@ -92,28 +105,31 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_TYPE Page)
 {
    PMM_ALLOCATION_REQUEST Request;
    PLIST_ENTRY Entry;
-   KIRQL oldIrql;
+   KIRQL OldIrql;
 
    if (Page == 0)
    {
       DPRINT1("Tried to release page zero.\n");
-      KEBUGCHECK(0);
+      KeBugCheck(MEMORY_MANAGEMENT);
    }
 
-   KeAcquireSpinLock(&AllocationListLock, &oldIrql);
+   KeAcquireSpinLock(&AllocationListLock, &OldIrql);
    if (MmGetReferenceCountPage(Page) == 1)
    {
       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-      if (IsListEmpty(&AllocationListHead) || MmStats.NrFreePages < MiMinimumAvailablePages)
+      if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
       {
-         KeReleaseSpinLock(&AllocationListLock, oldIrql);
+         KeReleaseSpinLock(&AllocationListLock, OldIrql);
+         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
+         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
          MmDereferencePage(Page);
+         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
       }
       else
       {
          Entry = RemoveHeadList(&AllocationListHead);
          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
-         KeReleaseSpinLock(&AllocationListLock, oldIrql);
+         KeReleaseSpinLock(&AllocationListLock, OldIrql);
          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
          MiZeroPage(Page);
          Request->Page = Page;
@@ -122,9 +138,11 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_TYPE Page)
    }
    else
    {
-      KeReleaseSpinLock(&AllocationListLock, oldIrql);
+      KeReleaseSpinLock(&AllocationListLock, OldIrql);
       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
+      OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
       MmDereferencePage(Page);
+      KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
    }
 
    return(STATUS_SUCCESS);
@@ -150,6 +168,33 @@ MiTrimMemoryConsumer(ULONG Consumer)
    }
 }
 
+NTSTATUS
+MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
+{
+    PFN_TYPE CurrentPage;
+    PFN_TYPE NextPage;
+    NTSTATUS Status;
+    
+    (*NrFreedPages) = 0;
+    
+    CurrentPage = MmGetLRUFirstUserPage();
+    while (CurrentPage != 0 && Target > 0)
+    {
+        NextPage = MmGetLRUNextUserPage(CurrentPage);
+        
+        Status = MmPageOutPhysicalAddress(CurrentPage);
+        if (NT_SUCCESS(Status))
+        {
+            DPRINT("Succeeded\n");
+            Target--;
+            (*NrFreedPages)++;
+        }
+        
+        CurrentPage = NextPage;
+    }
+    return(STATUS_SUCCESS);
+}
+
 VOID
 NTAPI
 MmRebalanceMemoryConsumers(VOID)
@@ -159,7 +204,7 @@ MmRebalanceMemoryConsumers(VOID)
    ULONG NrFreedPages;
    NTSTATUS Status;
 
-   Target = (MiMinimumAvailablePages - MmStats.NrFreePages) + MiPagesRequired;
+   Target = (MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 
    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
@@ -169,7 +214,7 @@ MmRebalanceMemoryConsumers(VOID)
          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
          if (!NT_SUCCESS(Status))
          {
-            KEBUGCHECK(0);
+            KeBugCheck(MEMORY_MANAGEMENT);
          }
          Target = Target - NrFreedPages;
       }
@@ -183,6 +228,8 @@ MiIsBalancerThread(VOID)
           PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
 }
 
+VOID NTAPI MiSetConsumer(IN PFN_TYPE Pfn, IN ULONG Consumer);
+
 NTSTATUS
 NTAPI
 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
@@ -190,7 +237,7 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 {
    ULONG OldUsed;
    PFN_TYPE Page;
-   KIRQL oldIrql;
+   KIRQL OldIrql;
 
    /*
     * Make sure we don't exceed our individual target.
@@ -210,15 +257,17 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
    /*
     * Allocate always memory for the non paged pool and for the pager thread.
     */
-   if (Consumer == MC_NPPOOL || MiIsBalancerThread())
+   if ((Consumer == MC_NPPOOL) || (Consumer == MC_SYSTEM) || MiIsBalancerThread())
    {
-      Page = MmAllocPage(Consumer, 0);
+      OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+      Page = MmAllocPage(Consumer);
+      KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
       if (Page == 0)
       {
-         KEBUGCHECK(NO_PAGES_AVAILABLE);
+         KeBugCheck(NO_PAGES_AVAILABLE);
       }
       *AllocatedPage = Page;
-      if (MmStats.NrFreePages <= MiMinimumAvailablePages &&
+      if (MmAvailablePages <= MiMinimumAvailablePages &&
             MiBalancerThreadHandle != NULL)
       {
          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
@@ -229,7 +278,7 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
    /*
     * Make sure we don't exceed global targets.
     */
-   if (MmStats.NrFreePages <= MiMinimumAvailablePages)
+   if (MmAvailablePages <= MiMinimumAvailablePages)
    {
       MM_ALLOCATION_REQUEST Request;
 
@@ -245,14 +294,14 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
       (void)InterlockedIncrementUL(&MiPagesRequired);
 
-      KeAcquireSpinLock(&AllocationListLock, &oldIrql);
+      KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 
       if (MiBalancerThreadHandle != NULL)
       {
          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
       }
       InsertTailList(&AllocationListHead, &Request.ListEntry);
-      KeReleaseSpinLock(&AllocationListLock, oldIrql);
+      KeReleaseSpinLock(&AllocationListLock, OldIrql);
 
       KeWaitForSingleObject(&Request.Event,
                             0,
@@ -263,10 +312,10 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
       Page = Request.Page;
       if (Page == 0)
       {
-         KEBUGCHECK(NO_PAGES_AVAILABLE);
+         KeBugCheck(NO_PAGES_AVAILABLE);
       }
-      /* Update the Consumer */
-      MiGetPfnEntry(Page)->Flags.Consumer = Consumer;
+      /* Update the Consumer and make the page active */
+      MiSetConsumer(Page, Consumer);
       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
       *AllocatedPage = Page;
       (void)InterlockedDecrementUL(&MiPagesRequired);
@@ -276,10 +325,12 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
    /*
     * Actually allocate the page.
     */
-   Page = MmAllocPage(Consumer, 0);
+   OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+   Page = MmAllocPage(Consumer);
+   KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
    if (Page == 0)
    {
-      KEBUGCHECK(NO_PAGES_AVAILABLE);
+      KeBugCheck(NO_PAGES_AVAILABLE);
    }
    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
    *AllocatedPage = Page;
@@ -287,7 +338,7 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
    return(STATUS_SUCCESS);
 }
 
-VOID STDCALL
+VOID NTAPI
 MiBalancerThread(PVOID Unused)
 {
    PVOID WaitObjects[2];
@@ -316,8 +367,7 @@ MiBalancerThread(PVOID Unused)
       if (Status == STATUS_SUCCESS)
       {
          /* MiBalancerEvent */
-         CHECKPOINT;
-         while (MmStats.NrFreePages < MiMinimumAvailablePages + 5)
+         while (MmAvailablePages < MiMinimumAvailablePages + 5)
          {
             for (i = 0; i < MC_MAXIMUM; i++)
             {
@@ -327,18 +377,17 @@ MiBalancerThread(PVOID Unused)
                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
                   if (!NT_SUCCESS(Status))
                   {
-                     KEBUGCHECK(0);
+                     KeBugCheck(MEMORY_MANAGEMENT);
                   }
                }
             }
          }
          InterlockedExchange(&MiBalancerWork, 0);
-         CHECKPOINT;
       }
       else if (Status == STATUS_SUCCESS + 1)
       {
          /* MiBalancerTimer */
-         ShouldRun = MmStats.NrFreePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
+         ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
          for (i = 0; i < MC_MAXIMUM; i++)
          {
             if (MiMemoryConsumers[i].Trim != NULL)
@@ -359,7 +408,7 @@ MiBalancerThread(PVOID Unused)
                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
                   if (!NT_SUCCESS(Status))
                   {
-                     KEBUGCHECK(0);
+                     KeBugCheck(MEMORY_MANAGEMENT);
                   }
                }
             }
@@ -368,7 +417,7 @@ MiBalancerThread(PVOID Unused)
       else
       {
          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
-         KEBUGCHECK(0);
+         KeBugCheck(MEMORY_MANAGEMENT);
       }
    }
 }
@@ -387,7 +436,6 @@ MiInitBalancerThread(VOID)
    ;
 #endif
 
-   CHECKPOINT;
 
    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
@@ -409,7 +457,7 @@ MiInitBalancerThread(VOID)
                                  NULL);
    if (!NT_SUCCESS(Status))
    {
-      KEBUGCHECK(0);
+      KeBugCheck(MEMORY_MANAGEMENT);
    }
 
    Priority = LOW_REALTIME_PRIORITY + 1;