[NTOSKRNL]
[reactos.git] / reactos / ntoskrnl / mm / balance.c
index 4835a4d..7e1a3fb 100644 (file)
@@ -5,6 +5,7 @@
  * PURPOSE:         kernel memory managment functions
  *
  * PROGRAMMERS:     David Welch (welch@cwcom.net)
+ *                  Cameron Gutman (cameron.gutman@reactos.org)
  */
 
 /* INCLUDES *****************************************************************/
@@ -13,6 +14,8 @@
 #define NDEBUG
 #include <debug.h>
 
+#include "ARM3/miarm.h"
+
 #if defined (ALLOC_PRAGMA)
 #pragma alloc_text(INIT, MmInitializeBalancer)
 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
 /* TYPES ********************************************************************/
 typedef struct _MM_ALLOCATION_REQUEST
 {
-   PFN_NUMBER Page;
-   LIST_ENTRY ListEntry;
-   KEVENT Event;
+    PFN_NUMBER Page;
+    LIST_ENTRY ListEntry;
+    KEVENT Event;
 }
 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
-
 /* GLOBALS ******************************************************************/
 
 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
@@ -36,14 +38,12 @@ static ULONG MiMinimumAvailablePages;
 static ULONG MiNrTotalPages;
 static LIST_ENTRY AllocationListHead;
 static KSPIN_LOCK AllocationListLock;
-static ULONG MiPagesRequired = 0;
-static ULONG MiMinimumPagesPerRun = 10;
+static ULONG MiMinimumPagesPerRun;
 
 static CLIENT_ID MiBalancerThreadId;
 static HANDLE MiBalancerThreadHandle = NULL;
 static KEVENT MiBalancerEvent;
 static KTIMER MiBalancerTimer;
-static LONG MiBalancerWork = 0;
 
 /* FUNCTIONS ****************************************************************/
 
@@ -52,14 +52,15 @@ INIT_FUNCTION
 NTAPI
 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
 {
-   memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
-   InitializeListHead(&AllocationListHead);
-   KeInitializeSpinLock(&AllocationListLock);
+    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
+    InitializeListHead(&AllocationListHead);
+    KeInitializeSpinLock(&AllocationListLock);
 
-   MiNrTotalPages = NrAvailablePages;
+    MiNrTotalPages = NrAvailablePages;
 
-   /* Set up targets. */
-   MiMinimumAvailablePages = 64;
+    /* Set up targets. */
+    MiMinimumAvailablePages = 128;
+    MiMinimumPagesPerRun = 256;
     if ((NrAvailablePages + NrSystemPages) >= 8192)
     {
         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
@@ -72,17 +73,17 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
     {
         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
     }
-   MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
+    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
 }
 
 VOID
 INIT_FUNCTION
 NTAPI
-MmInitializeMemoryConsumer(ULONG Consumer,
-                           NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
-                                            PULONG NrFreed))
+MmInitializeMemoryConsumer(
+    ULONG Consumer,
+    NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
 {
-   MiMemoryConsumers[Consumer].Trim = Trim;
+    MiMemoryConsumers[Consumer].Trim = Trim;
 }
 
 VOID
@@ -95,69 +96,103 @@ NTSTATUS
 NTAPI
 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
 {
-   PMM_ALLOCATION_REQUEST Request;
-   PLIST_ENTRY Entry;
-   KIRQL OldIrql;
-
-   if (Page == 0)
-   {
-      DPRINT1("Tried to release page zero.\n");
-      KeBugCheck(MEMORY_MANAGEMENT);
-   }
-
-   KeAcquireSpinLock(&AllocationListLock, &OldIrql);
-   if (MmGetReferenceCountPage(Page) == 1)
-   {
-      (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-      if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
-      {
-         KeReleaseSpinLock(&AllocationListLock, OldIrql);
-         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
-         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-         MmDereferencePage(Page);
-         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-      }
-      else
-      {
-         Entry = RemoveHeadList(&AllocationListHead);
-         Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
-         KeReleaseSpinLock(&AllocationListLock, OldIrql);
-         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
-         MiZeroPhysicalPage(Page);
-         Request->Page = Page;
-         KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
-      }
-   }
-   else
-   {
-      KeReleaseSpinLock(&AllocationListLock, OldIrql);
-      if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
-      OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-      MmDereferencePage(Page);
-      KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-   }
-
-   return(STATUS_SUCCESS);
+    PMM_ALLOCATION_REQUEST Request;
+    PLIST_ENTRY Entry;
+    KIRQL OldIrql;
+
+    if (Page == 0)
+    {
+        DPRINT1("Tried to release page zero.\n");
+        KeBugCheck(MEMORY_MANAGEMENT);
+    }
+
+    if (MmGetReferenceCountPage(Page) == 1)
+    {
+        if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
+        (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
+        if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
+        {
+            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+            MmDereferencePage(Page);
+            KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+        }
+        else
+        {
+            Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
+            MiZeroPhysicalPage(Page);
+            Request->Page = Page;
+            KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
+        }
+    }
+    else
+    {
+        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        MmDereferencePage(Page);
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+    }
+
+    return(STATUS_SUCCESS);
 }
 
-VOID
+ULONG
 NTAPI
-MiTrimMemoryConsumer(ULONG Consumer)
+MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 {
-   LONG Target;
-   ULONG NrFreedPages;
-
-   Target = MiMemoryConsumers[Consumer].PagesUsed -
-            MiMemoryConsumers[Consumer].PagesTarget;
-   if (Target < 1)
-   {
-      Target = 1;
-   }
-
-   if (MiMemoryConsumers[Consumer].Trim != NULL)
-   {
-      MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
-   }
+    ULONG Target = InitialTarget;
+    ULONG NrFreedPages = 0;
+    NTSTATUS Status;
+
+    /* Make sure we can trim this consumer */
+    if (!MiMemoryConsumers[Consumer].Trim)
+    {
+        /* Return the unmodified initial target */
+        return InitialTarget;
+    }
+
+    if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
+    {
+        /* Consumer page limit exceeded */
+        Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
+    }
+    if (MmAvailablePages < MiMinimumAvailablePages)
+    {
+        /* Global page limit exceeded */
+        Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
+    }
+
+    if (Target)
+    {
+        if (!InitialTarget)
+        {
+            /* If there was no initial target,
+             * swap at least MiMinimumPagesPerRun */
+            Target = max(Target, MiMinimumPagesPerRun);
+        }
+
+        /* Now swap the pages out */
+        Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
+
+        DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
+
+        if (!NT_SUCCESS(Status))
+        {
+            KeBugCheck(MEMORY_MANAGEMENT);
+        }
+
+        /* Update the target */
+        if (NrFreedPages < Target)
+            Target -= NrFreedPages;
+        else
+            Target = 0;
+
+        /* Return the remaining pages needed to meet the target */
+        return Target;
+    }
+    else
+    {
+        /* Initial target is zero and we don't have anything else to add */
+        return 0;
+    }
 }
 
 NTSTATUS
@@ -166,14 +201,12 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
     PFN_NUMBER CurrentPage;
     PFN_NUMBER NextPage;
     NTSTATUS Status;
-    
+
     (*NrFreedPages) = 0;
-    
+
     CurrentPage = MmGetLRUFirstUserPage();
     while (CurrentPage != 0 && Target > 0)
     {
-        NextPage = MmGetLRUNextUserPage(CurrentPage);
-        
         Status = MmPageOutPhysicalAddress(CurrentPage);
         if (NT_SUCCESS(Status))
         {
@@ -181,43 +214,42 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
             Target--;
             (*NrFreedPages)++;
         }
-        
+
+        NextPage = MmGetLRUNextUserPage(CurrentPage);
+        if (NextPage <= CurrentPage)
+        {
+            /* We wrapped around, so we're done */
+            break;
+        }
         CurrentPage = NextPage;
     }
-    return(STATUS_SUCCESS);
-}
 
-VOID
-NTAPI
-MmRebalanceMemoryConsumers(VOID)
-{
-   LONG Target;
-   ULONG i;
-   ULONG NrFreedPages;
-   NTSTATUS Status;
-
-   Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
-   Target = max(Target, (LONG) MiMinimumPagesPerRun);
-
-   for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
-   {
-      if (MiMemoryConsumers[i].Trim != NULL)
-      {
-         Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
-         if (!NT_SUCCESS(Status))
-         {
-            KeBugCheck(MEMORY_MANAGEMENT);
-         }
-         Target = Target - NrFreedPages;
-      }
-   }
+    return STATUS_SUCCESS;
 }
 
 static BOOLEAN
 MiIsBalancerThread(VOID)
 {
-   return MiBalancerThreadHandle != NULL &&
-          PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
+    return (MiBalancerThreadHandle != NULL) &&
+           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
+}
+
+VOID
+NTAPI
+MiDeletePte(IN PMMPTE PointerPte,
+            IN PVOID VirtualAddress,
+            IN PEPROCESS CurrentProcess,
+            IN PMMPTE PrototypePte);
+
+VOID
+NTAPI
+MmRebalanceMemoryConsumers(VOID)
+{
+    if (MiBalancerThreadHandle != NULL &&
+        !MiIsBalancerThread())
+    {
+        KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
+    }
 }
 
 NTSTATUS
@@ -225,190 +257,182 @@ NTAPI
 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
                             PPFN_NUMBER AllocatedPage)
 {
-   ULONG OldUsed;
-   PFN_NUMBER Page;
-   KIRQL OldIrql;
-
-   /*
-    * Make sure we don't exceed our individual target.
-    */
-   OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-   if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
-         !MiIsBalancerThread())
-   {
-      if (!CanWait)
-      {
-         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-         return(STATUS_NO_MEMORY);
-      }
-      MiTrimMemoryConsumer(Consumer);
-   }
-
-   /*
-    * Allocate always memory for the non paged pool and for the pager thread.
-    */
-   if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
-   {
-      OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-      Page = MmAllocPage(Consumer);
-      KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-      if (Page == 0)
-      {
-         KeBugCheck(NO_PAGES_AVAILABLE);
-      }
-      *AllocatedPage = Page;
-      if (MmAvailablePages <= MiMinimumAvailablePages &&
-            MiBalancerThreadHandle != NULL)
-      {
-         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
-      }
-      return(STATUS_SUCCESS);
-   }
-
-   /*
-    * Make sure we don't exceed global targets.
-    */
-   if (MmAvailablePages <= MiMinimumAvailablePages)
-   {
-      MM_ALLOCATION_REQUEST Request;
-
-      if (!CanWait)
-      {
-         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
-         return(STATUS_NO_MEMORY);
-      }
-
-      /* Insert an allocation request. */
-      Request.Page = 0;
-
-      KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
-      (void)InterlockedIncrementUL(&MiPagesRequired);
-
-      KeAcquireSpinLock(&AllocationListLock, &OldIrql);
-
-      if (MiBalancerThreadHandle != NULL)
-      {
-         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
-      }
-      InsertTailList(&AllocationListHead, &Request.ListEntry);
-      KeReleaseSpinLock(&AllocationListLock, OldIrql);
-
-      KeWaitForSingleObject(&Request.Event,
-                            0,
-                            KernelMode,
-                            FALSE,
-                            NULL);
-
-      Page = Request.Page;
-      if (Page == 0)
-      {
-         KeBugCheck(NO_PAGES_AVAILABLE);
-      }
-      /* Update the Consumer and make the page active */
-      if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
-      *AllocatedPage = Page;
-      (void)InterlockedDecrementUL(&MiPagesRequired);
-      return(STATUS_SUCCESS);
-   }
-
-   /*
-    * Actually allocate the page.
-    */
-   OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
-   Page = MmAllocPage(Consumer);
-   KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-   if (Page == 0)
-   {
-      KeBugCheck(NO_PAGES_AVAILABLE);
-   }
-   if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
-   *AllocatedPage = Page;
-
-   return(STATUS_SUCCESS);
+    ULONG PagesUsed;
+    PFN_NUMBER Page;
+    KIRQL OldIrql;
+
+    /*
+     * Make sure we don't exceed our individual target.
+     */
+    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
+    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
+            !MiIsBalancerThread())
+    {
+        MmRebalanceMemoryConsumers();
+    }
+
+    /*
+     * Allocate always memory for the non paged pool and for the pager thread.
+     */
+    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
+    {
+        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        Page = MmAllocPage(Consumer);
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+        if (Page == 0)
+        {
+            KeBugCheck(NO_PAGES_AVAILABLE);
+        }
+        if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
+        *AllocatedPage = Page;
+        if (MmAvailablePages < MiMinimumAvailablePages)
+            MmRebalanceMemoryConsumers();
+        return(STATUS_SUCCESS);
+    }
+
+    /*
+     * Make sure we don't exceed global targets.
+     */
+    if (MmAvailablePages < MiMinimumAvailablePages)
+    {
+        MM_ALLOCATION_REQUEST Request;
+
+        if (!CanWait)
+        {
+            (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
+            MmRebalanceMemoryConsumers();
+            return(STATUS_NO_MEMORY);
+        }
+
+        /* Insert an allocation request. */
+        Request.Page = 0;
+        KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
+
+        ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
+        MmRebalanceMemoryConsumers();
+
+        KeWaitForSingleObject(&Request.Event,
+                              0,
+                              KernelMode,
+                              FALSE,
+                              NULL);
+
+        Page = Request.Page;
+        if (Page == 0)
+        {
+            KeBugCheck(NO_PAGES_AVAILABLE);
+        }
+
+        if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
+        *AllocatedPage = Page;
+
+        if (MmAvailablePages < MiMinimumAvailablePages)
+        {
+            MmRebalanceMemoryConsumers();
+        }
+
+        return(STATUS_SUCCESS);
+    }
+
+    /*
+     * Actually allocate the page.
+     */
+    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+    Page = MmAllocPage(Consumer);
+    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+    if (Page == 0)
+    {
+        KeBugCheck(NO_PAGES_AVAILABLE);
+    }
+    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
+    *AllocatedPage = Page;
+
+    if (MmAvailablePages < MiMinimumAvailablePages)
+    {
+        MmRebalanceMemoryConsumers();
+    }
+
+    return(STATUS_SUCCESS);
 }
 
+
 VOID NTAPI
 MiBalancerThread(PVOID Unused)
 {
-   PVOID WaitObjects[2];
-   NTSTATUS Status;
-   ULONG i;
-   ULONG NrFreedPages;
-   ULONG NrPagesUsed;
-   ULONG Target;
-   BOOLEAN ShouldRun;
-
-
-   WaitObjects[0] = &MiBalancerEvent;
-   WaitObjects[1] = &MiBalancerTimer;
-
-   while (1)
-   {
-      Status = KeWaitForMultipleObjects(2,
-                                        WaitObjects,
-                                        WaitAny,
-                                        Executive,
-                                        KernelMode,
-                                        FALSE,
-                                        NULL,
-                                        NULL);
-
-      if (Status == STATUS_SUCCESS)
-      {
-         /* MiBalancerEvent */
-         while (MmAvailablePages < MiMinimumAvailablePages + 5)
-         {
-            for (i = 0; i < MC_MAXIMUM; i++)
+    PVOID WaitObjects[2];
+    NTSTATUS Status;
+    ULONG i;
+
+    WaitObjects[0] = &MiBalancerEvent;
+    WaitObjects[1] = &MiBalancerTimer;
+
+    while (1)
+    {
+        Status = KeWaitForMultipleObjects(2,
+                                          WaitObjects,
+                                          WaitAny,
+                                          Executive,
+                                          KernelMode,
+                                          FALSE,
+                                          NULL,
+                                          NULL);
+
+        if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
+        {
+            ULONG InitialTarget = 0;
+
+#if (_MI_PAGING_LEVELS == 2)
+            if (!MiIsBalancerThread())
             {
-               if (MiMemoryConsumers[i].Trim != NULL)
-               {
-                  NrFreedPages = 0;
-                  Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
-                  if (!NT_SUCCESS(Status))
-                  {
-                     KeBugCheck(MEMORY_MANAGEMENT);
-                  }
-               }
+                /* Clean up the unused PDEs */
+                ULONG_PTR Address;
+                PEPROCESS Process = PsGetCurrentProcess();
+
+                /* Acquire PFN lock */
+                KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+                PMMPDE pointerPde;
+                for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
+                        Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
+                        Address += (PAGE_SIZE * PTE_COUNT))
+                {
+                    if (MiQueryPageTableReferences((PVOID)Address) == 0)
+                    {
+                        pointerPde = MiAddressToPde(Address);
+                        if (pointerPde->u.Hard.Valid)
+                            MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
+                        ASSERT(pointerPde->u.Hard.Valid == 0);
+                    }
+                }
+                /* Release lock */
+                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
             }
-         }
-         InterlockedExchange(&MiBalancerWork, 0);
-      }
-      else if (Status == STATUS_SUCCESS + 1)
-      {
-         /* MiBalancerTimer */
-         ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
-         for (i = 0; i < MC_MAXIMUM; i++)
-         {
-            if (MiMemoryConsumers[i].Trim != NULL)
+#endif
+            do
             {
-               NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
-               if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
-               {
-                  if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
-                  {
-                     Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
-                                   MiMinimumPagesPerRun);
-                  }
-                  else
-                  {
-                     Target = MiMinimumPagesPerRun;
-                  }
-                  NrFreedPages = 0;
-                  Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
-                  if (!NT_SUCCESS(Status))
-                  {
-                     KeBugCheck(MEMORY_MANAGEMENT);
-                  }
-               }
+                ULONG OldTarget = InitialTarget;
+
+                /* Trim each consumer */
+                for (i = 0; i < MC_MAXIMUM; i++)
+                {
+                    InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
+                }
+
+                /* No pages left to swap! */
+                if (InitialTarget != 0 &&
+                        InitialTarget == OldTarget)
+                {
+                    /* Game over */
+                    KeBugCheck(NO_PAGES_AVAILABLE);
+                }
             }
-         }
-      }
-      else
-      {
-         DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
-         KeBugCheck(MEMORY_MANAGEMENT);
-      }
-   }
+            while (InitialTarget != 0);
+        }
+        else
+        {
+            DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
+            KeBugCheck(MEMORY_MANAGEMENT);
+        }
+    }
 }
 
 VOID
@@ -416,44 +440,44 @@ INIT_FUNCTION
 NTAPI
 MiInitBalancerThread(VOID)
 {
-   KPRIORITY Priority;
-   NTSTATUS Status;
+    KPRIORITY Priority;
+    NTSTATUS Status;
 #if !defined(__GNUC__)
 
-   LARGE_INTEGER dummyJunkNeeded;
-   dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
-   ;
+    LARGE_INTEGER dummyJunkNeeded;
+    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
+    ;
 #endif
 
 
-   KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
-   KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
-   KeSetTimerEx(&MiBalancerTimer,
+    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
+    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
+    KeSetTimerEx(&MiBalancerTimer,
 #if defined(__GNUC__)
-                (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
+                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 #else
-                dummyJunkNeeded,
+                 dummyJunkNeeded,
 #endif
-                2000,         /* 2 sec */
-                NULL);
-
-   Status = PsCreateSystemThread(&MiBalancerThreadHandle,
-                                 THREAD_ALL_ACCESS,
-                                 NULL,
-                                 NULL,
-                                 &MiBalancerThreadId,
-                                 (PKSTART_ROUTINE) MiBalancerThread,
-                                 NULL);
-   if (!NT_SUCCESS(Status))
-   {
-      KeBugCheck(MEMORY_MANAGEMENT);
-   }
-
-   Priority = LOW_REALTIME_PRIORITY + 1;
-   NtSetInformationThread(MiBalancerThreadHandle,
-                          ThreadPriority,
-                          &Priority,
-                          sizeof(Priority));
+                 2000,         /* 2 sec */
+                 NULL);
+
+    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
+                                  THREAD_ALL_ACCESS,
+                                  NULL,
+                                  NULL,
+                                  &MiBalancerThreadId,
+                                  MiBalancerThread,
+                                  NULL);
+    if (!NT_SUCCESS(Status))
+    {
+        KeBugCheck(MEMORY_MANAGEMENT);
+    }
+
+    Priority = LOW_REALTIME_PRIORITY + 1;
+    NtSetInformationThread(MiBalancerThreadHandle,
+                           ThreadPriority,
+                           &Priority,
+                           sizeof(Priority));
 
 }