ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 #if defined (ALLOC_PRAGMA)
  20 #pragma alloc_text(INIT, MmInitializeBalancer)
  21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  22 #pragma alloc_text(INIT, MiInitBalancerThread)
  23 #endif
  24
  25
  26 /* TYPES ********************************************************************/
  27 typedef struct _MM_ALLOCATION_REQUEST
  28 {
  29    PFN_NUMBER Page;
  30    LIST_ENTRY ListEntry;
  31    KEVENT Event;
  32 }
  33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  34 /* GLOBALS ******************************************************************/
  35
  36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  37 static ULONG MiMinimumAvailablePages;
  38 static ULONG MiNrTotalPages;
  39 static LIST_ENTRY AllocationListHead;
  40 static KSPIN_LOCK AllocationListLock;
  41 static ULONG MiMinimumPagesPerRun;
  42
  43 static CLIENT_ID MiBalancerThreadId;
  44 static HANDLE MiBalancerThreadHandle = NULL;
  45 static KEVENT MiBalancerEvent;
  46 static KTIMER MiBalancerTimer;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 128;
  63    MiMinimumPagesPerRun = 256;
  64     if ((NrAvailablePages + NrSystemPages) >= 8192)
  65     {
  66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  67     }
  68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  69     {
  70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  71     }
  72     else
  73     {
  74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  75     }
  76    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  77 }
  78
  79 VOID
  80 INIT_FUNCTION
  81 NTAPI
  82 MmInitializeMemoryConsumer(ULONG Consumer,
  83                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  84                                             PULONG NrFreed))
  85 {
  86    MiMemoryConsumers[Consumer].Trim = Trim;
  87 }
  88
  89 VOID
  90 NTAPI
  91 MiZeroPhysicalPage(
  92     IN PFN_NUMBER PageFrameIndex
  93 );
  94
  95 NTSTATUS
  96 NTAPI
  97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  98 {
  99    PMM_ALLOCATION_REQUEST Request;
 100    PLIST_ENTRY Entry;
 101    KIRQL OldIrql;
 102
 103    if (Page == 0)
 104    {
 105       DPRINT1("Tried to release page zero.\n");
 106       KeBugCheck(MEMORY_MANAGEMENT);
 107    }
 108
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 112       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 113       if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 114       {
 115          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 116          MmDereferencePage(Page);
 117          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 118       }
 119       else
 120       {
 121          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 122          MiZeroPhysicalPage(Page);
 123          Request->Page = Page;
 124          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 125       }
 126    }
 127    else
 128    {
 129       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 130       MmDereferencePage(Page);
 131       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 132    }
 133
 134    return(STATUS_SUCCESS);
 135 }
 136
 137 ULONG
 138 NTAPI
 139 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 140 {
 141     ULONG Target = InitialTarget;
 142     ULONG NrFreedPages = 0;
 143     NTSTATUS Status;
 144
 145     /* Make sure we can trim this consumer */
 146     if (!MiMemoryConsumers[Consumer].Trim)
 147     {
 148         /* Return the unmodified initial target */
 149         return InitialTarget;
 150     }
 151
 152     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 153     {
 154         /* Consumer page limit exceeded */
 155         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 156     }
 157     if (MmAvailablePages < MiMinimumAvailablePages)
 158     {
 159         /* Global page limit exceeded */
 160         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
 161     }
 162
 163     if (Target)
 164     {
 165         if (!InitialTarget)
 166         {
 167             /* If there was no initial target,
 168              * swap at least MiMinimumPagesPerRun */
 169             Target = max(Target, MiMinimumPagesPerRun);
 170         }
 171
 172         /* Now swap the pages out */
 173         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 174
 175         DPRINT("Trimming consumer %d: Freed %d pages with a target of %d pages\n", Consumer, NrFreedPages, Target);
 176
 177         if (!NT_SUCCESS(Status))
 178         {
 179             KeBugCheck(MEMORY_MANAGEMENT);
 180         }
 181
 182         /* Update the target */
 183         if (NrFreedPages < Target)
 184             Target -= NrFreedPages;
 185         else
 186             Target = 0;
 187
 188         /* Return the remaining pages needed to meet the target */
 189         return Target;
 190     }
 191     else
 192     {
 193         /* Initial target is zero and we don't have anything else to add */
 194         return 0;
 195     }
 196 }
 197
 198 NTSTATUS
 199 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 200 {
 201     PFN_NUMBER CurrentPage;
 202     PFN_NUMBER NextPage;
 203     NTSTATUS Status;
 204
 205     (*NrFreedPages) = 0;
 206
 207     CurrentPage = MmGetLRUFirstUserPage();
 208     while (CurrentPage != 0 && Target > 0)
 209     {
 210         Status = MmPageOutPhysicalAddress(CurrentPage);
 211         if (NT_SUCCESS(Status))
 212         {
 213             DPRINT("Succeeded\n");
 214             Target--;
 215             (*NrFreedPages)++;
 216         }
 217
 218         NextPage = MmGetLRUNextUserPage(CurrentPage);
 219         if (NextPage <= CurrentPage)
 220         {
 221             /* We wrapped around, so we're done */
 222             break;
 223         }
 224         CurrentPage = NextPage;
 225     }
 226
 227     return STATUS_SUCCESS;
 228 }
 229
 230 static BOOLEAN
 231 MiIsBalancerThread(VOID)
 232 {
 233    return (MiBalancerThreadHandle != NULL) &&
 234           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 235 }
 236
 237 VOID
 238 NTAPI
 239 MiDeletePte(IN PMMPTE PointerPte,
 240             IN PVOID VirtualAddress,
 241             IN PEPROCESS CurrentProcess,
 242             IN PMMPTE PrototypePte);
 243
 244 VOID
 245 NTAPI
 246 MmRebalanceMemoryConsumers(VOID)
 247 {
 248     if (MiBalancerThreadHandle != NULL &&
 249         !MiIsBalancerThread())
 250     {
 251         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 252     }
 253 }
 254
 255 NTSTATUS
 256 NTAPI
 257 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 258                             PPFN_NUMBER AllocatedPage)
 259 {
 260    ULONG PagesUsed;
 261    PFN_NUMBER Page;
 262    KIRQL OldIrql;
 263
 264    /*
 265     * Make sure we don't exceed our individual target.
 266     */
 267    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 268    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 269        !MiIsBalancerThread())
 270    {
 271       MmRebalanceMemoryConsumers();
 272    }
 273
 274    /*
 275     * Allocate always memory for the non paged pool and for the pager thread.
 276     */
 277    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 278    {
 279       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 280       Page = MmAllocPage(Consumer);
 281       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 282       if (Page == 0)
 283       {
 284          KeBugCheck(NO_PAGES_AVAILABLE);
 285       }
 286       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 287       *AllocatedPage = Page;
 288       if (MmAvailablePages < MiMinimumAvailablePages)
 289           MmRebalanceMemoryConsumers();
 290       return(STATUS_SUCCESS);
 291    }
 292
 293    /*
 294     * Make sure we don't exceed global targets.
 295     */
 296    if (MmAvailablePages < MiMinimumAvailablePages)
 297    {
 298       MM_ALLOCATION_REQUEST Request;
 299
 300       if (!CanWait)
 301       {
 302          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 303          MmRebalanceMemoryConsumers();
 304          return(STATUS_NO_MEMORY);
 305       }
 306
 307       /* Insert an allocation request. */
 308       Request.Page = 0;
 309       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 310
 311       ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 312       MmRebalanceMemoryConsumers();
 313
 314       KeWaitForSingleObject(&Request.Event,
 315                             0,
 316                             KernelMode,
 317                             FALSE,
 318                             NULL);
 319
 320       Page = Request.Page;
 321       if (Page == 0)
 322       {
 323          KeBugCheck(NO_PAGES_AVAILABLE);
 324       }
 325
 326       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 327       *AllocatedPage = Page;
 328
 329       if (MmAvailablePages < MiMinimumAvailablePages)
 330       {
 331           MmRebalanceMemoryConsumers();
 332       }
 333
 334       return(STATUS_SUCCESS);
 335    }
 336
 337    /*
 338     * Actually allocate the page.
 339     */
 340    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 341    Page = MmAllocPage(Consumer);
 342    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 343    if (Page == 0)
 344    {
 345       KeBugCheck(NO_PAGES_AVAILABLE);
 346    }
 347    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 348    *AllocatedPage = Page;
 349
 350    if (MmAvailablePages < MiMinimumAvailablePages)
 351    {
 352        MmRebalanceMemoryConsumers();
 353    }
 354
 355    return(STATUS_SUCCESS);
 356 }
 357
 358 VOID NTAPI
 359 MiBalancerThread(PVOID Unused)
 360 {
 361    PVOID WaitObjects[2];
 362    NTSTATUS Status;
 363    ULONG i;
 364
 365    WaitObjects[0] = &MiBalancerEvent;
 366    WaitObjects[1] = &MiBalancerTimer;
 367
 368    while (1)
 369    {
 370       Status = KeWaitForMultipleObjects(2,
 371                                         WaitObjects,
 372                                         WaitAny,
 373                                         Executive,
 374                                         KernelMode,
 375                                         FALSE,
 376                                         NULL,
 377                                         NULL);
 378
 379       if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 380       {
 381         ULONG InitialTarget = 0;
 382
 383 #if (_MI_PAGING_LEVELS == 2)
 384         if(!MiIsBalancerThread())
 385         {
 386             /* Clean up the unused PDEs */
 387             ULONG_PTR Address;
 388             PEPROCESS Process = PsGetCurrentProcess();
 389
 390             /* Acquire PFN lock */
 391             KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 392             PMMPDE pointerPde;
 393             for(Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 394                 Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 395                 Address += (PAGE_SIZE * PTE_COUNT))
 396             {
 397                 if(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] == 0)
 398                 {
 399                     pointerPde = MiAddressToPde(Address);
 400                     if(pointerPde->u.Hard.Valid)
 401                         MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 402                     ASSERT(pointerPde->u.Hard.Valid == 0);
 403                 }
 404             }
 405             /* Release lock */
 406             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 407         }
 408 #endif
 409           do
 410           {
 411               ULONG OldTarget = InitialTarget;
 412
 413               /* Trim each consumer */
 414               for (i = 0; i < MC_MAXIMUM; i++)
 415               {
 416                   InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 417               }
 418
 419               /* No pages left to swap! */
 420               if (InitialTarget != 0 &&
 421                   InitialTarget == OldTarget)
 422               {
 423                   /* Game over */
 424                   KeBugCheck(NO_PAGES_AVAILABLE);
 425               }
 426           } while (InitialTarget != 0);
 427       }
 428       else
 429       {
 430          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 431          KeBugCheck(MEMORY_MANAGEMENT);
 432       }
 433    }
 434 }
 435
 436 VOID
 437 INIT_FUNCTION
 438 NTAPI
 439 MiInitBalancerThread(VOID)
 440 {
 441    KPRIORITY Priority;
 442    NTSTATUS Status;
 443 #if !defined(__GNUC__)
 444
 445    LARGE_INTEGER dummyJunkNeeded;
 446    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 447    ;
 448 #endif
 449
 450
 451    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 452    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 453    KeSetTimerEx(&MiBalancerTimer,
 454 #if defined(__GNUC__)
 455                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 456 #else
 457                 dummyJunkNeeded,
 458 #endif
 459                 2000,         /* 2 sec */
 460                 NULL);
 461
 462    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 463                                  THREAD_ALL_ACCESS,
 464                                  NULL,
 465                                  NULL,
 466                                  &MiBalancerThreadId,
 467                                  (PKSTART_ROUTINE) MiBalancerThread,
 468                                  NULL);
 469    if (!NT_SUCCESS(Status))
 470    {
 471       KeBugCheck(MEMORY_MANAGEMENT);
 472    }
 473
 474    Priority = LOW_REALTIME_PRIORITY + 1;
 475    NtSetInformationThread(MiBalancerThreadHandle,
 476                           ThreadPriority,
 477                           &Priority,
 478                           sizeof(Priority));
 479
 480 }
 481
 482
 483 /* EOF */