reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 #if defined (ALLOC_PRAGMA)
  20 #pragma alloc_text(INIT, MmInitializeBalancer)
  21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  22 #pragma alloc_text(INIT, MiInitBalancerThread)
  23 #endif
  24
  25
  26 /* TYPES ********************************************************************/
  27 typedef struct _MM_ALLOCATION_REQUEST
  28 {
  29    PFN_NUMBER Page;
  30    LIST_ENTRY ListEntry;
  31    KEVENT Event;
  32 }
  33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  34 /* GLOBALS ******************************************************************/
  35
  36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  37 static ULONG MiMinimumAvailablePages;
  38 static ULONG MiNrTotalPages;
  39 static LIST_ENTRY AllocationListHead;
  40 static KSPIN_LOCK AllocationListLock;
  41 static ULONG MiMinimumPagesPerRun;
  42
  43 static CLIENT_ID MiBalancerThreadId;
  44 static HANDLE MiBalancerThreadHandle = NULL;
  45 static KEVENT MiBalancerEvent;
  46 static KTIMER MiBalancerTimer;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 128;
  63    MiMinimumPagesPerRun = 256;
  64     if ((NrAvailablePages + NrSystemPages) >= 8192)
  65     {
  66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  67     }
  68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  69     {
  70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  71     }
  72     else
  73     {
  74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  75     }
  76    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  77 }
  78
  79 VOID
  80 INIT_FUNCTION
  81 NTAPI
  82 MmInitializeMemoryConsumer(ULONG Consumer,
  83                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  84                                             PULONG NrFreed))
  85 {
  86    MiMemoryConsumers[Consumer].Trim = Trim;
  87 }
  88
  89 VOID
  90 NTAPI
  91 MiZeroPhysicalPage(
  92     IN PFN_NUMBER PageFrameIndex
  93 );
  94
  95 NTSTATUS
  96 NTAPI
  97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  98 {
  99    PMM_ALLOCATION_REQUEST Request;
 100    PLIST_ENTRY Entry;
 101    KIRQL OldIrql;
 102
 103    if (Page == 0)
 104    {
 105       DPRINT1("Tried to release page zero.\n");
 106       KeBugCheck(MEMORY_MANAGEMENT);
 107    }
 108
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 112       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 113       if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 114       {
 115          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 116          MmDereferencePage(Page);
 117          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 118       }
 119       else
 120       {
 121          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 122          MiZeroPhysicalPage(Page);
 123          Request->Page = Page;
 124          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 125       }
 126    }
 127    else
 128    {
 129       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 130       MmDereferencePage(Page);
 131       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 132    }
 133
 134    return(STATUS_SUCCESS);
 135 }
 136
 137 ULONG
 138 NTAPI
 139 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 140 {
 141     LONG Target = InitialTarget;
 142     ULONG NrFreedPages = 0;
 143     NTSTATUS Status;
 144
 145     /* Make sure we can trim this consumer */
 146     if (!MiMemoryConsumers[Consumer].Trim)
 147     {
 148         /* Return the unmodified initial target */
 149         return InitialTarget;
 150     }
 151
 152     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 153     {
 154         /* Consumer page limit exceeded */
 155         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 156     }
 157     if (MmAvailablePages < MiMinimumAvailablePages)
 158     {
 159         /* Global page limit exceeded */
 160         Target = max(Target, MiMinimumAvailablePages - MmAvailablePages);
 161     }
 162
 163     if (Target)
 164     {
 165         if (!InitialTarget)
 166         {
 167             /* If there was no initial target,
 168              * swap at least MiMinimumPagesPerRun */
 169             Target = max(Target, MiMinimumPagesPerRun);
 170         }
 171
 172         /* Now swap the pages out */
 173         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 174
 175         DPRINT("Trimming consumer %d: Freed %d pages with a target of %d pages\n", Consumer, NrFreedPages, Target);
 176
 177         if (!NT_SUCCESS(Status))
 178         {
 179             KeBugCheck(MEMORY_MANAGEMENT);
 180         }
 181
 182         /* Update the target */
 183         if (NrFreedPages < Target)
 184             Target -= NrFreedPages;
 185         else
 186             Target = 0;
 187
 188         /* Return the remaining pages needed to meet the target */
 189         return Target;
 190     }
 191     else
 192     {
 193         /* Initial target is zero and we don't have anything else to add */
 194         return 0;
 195     }
 196 }
 197
 198 NTSTATUS
 199 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 200 {
 201     PFN_NUMBER CurrentPage;
 202     PFN_NUMBER NextPage;
 203     NTSTATUS Status;
 204
 205     (*NrFreedPages) = 0;
 206
 207     CurrentPage = MmGetLRUFirstUserPage();
 208     while (CurrentPage != 0 && Target > 0)
 209     {
 210         Status = MmPageOutPhysicalAddress(CurrentPage);
 211         if (NT_SUCCESS(Status))
 212         {
 213             DPRINT("Succeeded\n");
 214             Target--;
 215             (*NrFreedPages)++;
 216         }
 217
 218         NextPage = MmGetLRUNextUserPage(CurrentPage);
 219         if (NextPage <= CurrentPage)
 220         {
 221             /* We wrapped around, so we're done */
 222             break;
 223         }
 224         CurrentPage = NextPage;
 225     }
 226
 227     return STATUS_SUCCESS;
 228 }
 229
 230 static BOOLEAN
 231 MiIsBalancerThread(VOID)
 232 {
 233    return (MiBalancerThreadHandle != NULL) &&
 234           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 235 }
 236
 237 VOID
 238 NTAPI
 239 MiDeletePte(IN PMMPTE PointerPte,
 240             IN PVOID VirtualAddress,
 241             IN PEPROCESS CurrentProcess,
 242             IN PMMPTE PrototypePte);
 243
 244 VOID
 245 NTAPI
 246 MmRebalanceMemoryConsumers(VOID)
 247 {
 248 #if (_MI_PAGING_LEVELS == 2)
 249     if(!MiIsBalancerThread())
 250     {
 251         /* Clean up the unused PDEs */
 252         ULONG_PTR Address;
 253         KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 254         PMMPDE pointerPde;
 255         for(Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 256             Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 257             Address += (PAGE_SIZE * PTE_COUNT))
 258         {
 259             if(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] == 0)
 260             {
 261                 pointerPde = MiAddressToPde(Address);
 262                 if(pointerPde->u.Hard.Valid)
 263                     MiDeletePte(pointerPde, MiPdeToPte(pointerPde), PsGetCurrentProcess(), NULL);
 264                 ASSERT(pointerPde->u.Hard.Valid == 0);
 265             }
 266         }
 267         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 268     }
 269 #endif
 270
 271     if (MiBalancerThreadHandle != NULL &&
 272         !MiIsBalancerThread())
 273     {
 274         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 275     }
 276 }
 277
 278 NTSTATUS
 279 NTAPI
 280 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 281                             PPFN_NUMBER AllocatedPage)
 282 {
 283    ULONG PagesUsed;
 284    PFN_NUMBER Page;
 285    KIRQL OldIrql;
 286
 287    /*
 288     * Make sure we don't exceed our individual target.
 289     */
 290    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 291    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 292        !MiIsBalancerThread())
 293    {
 294       MmRebalanceMemoryConsumers();
 295    }
 296
 297    /*
 298     * Allocate always memory for the non paged pool and for the pager thread.
 299     */
 300    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 301    {
 302       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 303       Page = MmAllocPage(Consumer);
 304       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 305       if (Page == 0)
 306       {
 307          KeBugCheck(NO_PAGES_AVAILABLE);
 308       }
 309       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 310       *AllocatedPage = Page;
 311       if (MmAvailablePages < MiMinimumAvailablePages)
 312           MmRebalanceMemoryConsumers();
 313       return(STATUS_SUCCESS);
 314    }
 315
 316    /*
 317     * Make sure we don't exceed global targets.
 318     */
 319    if (MmAvailablePages < MiMinimumAvailablePages)
 320    {
 321       MM_ALLOCATION_REQUEST Request;
 322
 323       if (!CanWait)
 324       {
 325          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 326          MmRebalanceMemoryConsumers();
 327          return(STATUS_NO_MEMORY);
 328       }
 329
 330       /* Insert an allocation request. */
 331       Request.Page = 0;
 332       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 333
 334       ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 335       MmRebalanceMemoryConsumers();
 336
 337       KeWaitForSingleObject(&Request.Event,
 338                             0,
 339                             KernelMode,
 340                             FALSE,
 341                             NULL);
 342
 343       Page = Request.Page;
 344       if (Page == 0)
 345       {
 346          KeBugCheck(NO_PAGES_AVAILABLE);
 347       }
 348
 349       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 350       *AllocatedPage = Page;
 351
 352       if (MmAvailablePages < MiMinimumAvailablePages)
 353       {
 354           MmRebalanceMemoryConsumers();
 355       }
 356
 357       return(STATUS_SUCCESS);
 358    }
 359
 360    /*
 361     * Actually allocate the page.
 362     */
 363    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 364    Page = MmAllocPage(Consumer);
 365    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 366    if (Page == 0)
 367    {
 368       KeBugCheck(NO_PAGES_AVAILABLE);
 369    }
 370    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 371    *AllocatedPage = Page;
 372
 373    if (MmAvailablePages < MiMinimumAvailablePages)
 374    {
 375        MmRebalanceMemoryConsumers();
 376    }
 377
 378    return(STATUS_SUCCESS);
 379 }
 380
 381 VOID NTAPI
 382 MiBalancerThread(PVOID Unused)
 383 {
 384    PVOID WaitObjects[2];
 385    NTSTATUS Status;
 386    ULONG i;
 387
 388    WaitObjects[0] = &MiBalancerEvent;
 389    WaitObjects[1] = &MiBalancerTimer;
 390
 391    while (1)
 392    {
 393       Status = KeWaitForMultipleObjects(2,
 394                                         WaitObjects,
 395                                         WaitAny,
 396                                         Executive,
 397                                         KernelMode,
 398                                         FALSE,
 399                                         NULL,
 400                                         NULL);
 401
 402       if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 403       {
 404           ULONG InitialTarget = 0;
 405
 406           do
 407           {
 408               ULONG OldTarget = InitialTarget;
 409
 410               /* Trim each consumer */
 411               for (i = 0; i < MC_MAXIMUM; i++)
 412               {
 413                   InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 414               }
 415
 416               /* No pages left to swap! */
 417               if (InitialTarget != 0 &&
 418                   InitialTarget == OldTarget)
 419               {
 420                   /* Game over */
 421                   KeBugCheck(NO_PAGES_AVAILABLE);
 422               }
 423           } while (InitialTarget != 0);
 424       }
 425       else
 426       {
 427          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 428          KeBugCheck(MEMORY_MANAGEMENT);
 429       }
 430    }
 431 }
 432
 433 VOID
 434 INIT_FUNCTION
 435 NTAPI
 436 MiInitBalancerThread(VOID)
 437 {
 438    KPRIORITY Priority;
 439    NTSTATUS Status;
 440 #if !defined(__GNUC__)
 441
 442    LARGE_INTEGER dummyJunkNeeded;
 443    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 444    ;
 445 #endif
 446
 447
 448    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 449    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 450    KeSetTimerEx(&MiBalancerTimer,
 451 #if defined(__GNUC__)
 452                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 453 #else
 454                 dummyJunkNeeded,
 455 #endif
 456                 2000,         /* 2 sec */
 457                 NULL);
 458
 459    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 460                                  THREAD_ALL_ACCESS,
 461                                  NULL,
 462                                  NULL,
 463                                  &MiBalancerThreadId,
 464                                  (PKSTART_ROUTINE) MiBalancerThread,
 465                                  NULL);
 466    if (!NT_SUCCESS(Status))
 467    {
 468       KeBugCheck(MEMORY_MANAGEMENT);
 469    }
 470
 471    Priority = LOW_REALTIME_PRIORITY + 1;
 472    NtSetInformationThread(MiBalancerThreadHandle,
 473                           ThreadPriority,
 474                           &Priority,
 475                           sizeof(Priority));
 476
 477 }
 478
 479
 480 /* EOF */