reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 #if defined (ALLOC_PRAGMA)
  20 #pragma alloc_text(INIT, MmInitializeBalancer)
  21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  22 #pragma alloc_text(INIT, MiInitBalancerThread)
  23 #endif
  24
  25
  26 /* TYPES ********************************************************************/
  27 typedef struct _MM_ALLOCATION_REQUEST
  28 {
  29    PFN_NUMBER Page;
  30    LIST_ENTRY ListEntry;
  31    KEVENT Event;
  32 }
  33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  34 /* GLOBALS ******************************************************************/
  35
  36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  37 static ULONG MiMinimumAvailablePages;
  38 static ULONG MiNrTotalPages;
  39 static LIST_ENTRY AllocationListHead;
  40 static KSPIN_LOCK AllocationListLock;
  41 static ULONG MiMinimumPagesPerRun;
  42
  43 static CLIENT_ID MiBalancerThreadId;
  44 static HANDLE MiBalancerThreadHandle = NULL;
  45 static KEVENT MiBalancerEvent;
  46 static KTIMER MiBalancerTimer;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 128;
  63    MiMinimumPagesPerRun = 256;
  64     if ((NrAvailablePages + NrSystemPages) >= 8192)
  65     {
  66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  67     }
  68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  69     {
  70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  71     }
  72     else
  73     {
  74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  75     }
  76    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  77 }
  78
  79 VOID
  80 INIT_FUNCTION
  81 NTAPI
  82 MmInitializeMemoryConsumer(ULONG Consumer,
  83                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  84                                             PULONG NrFreed))
  85 {
  86    MiMemoryConsumers[Consumer].Trim = Trim;
  87 }
  88
  89 VOID
  90 NTAPI
  91 MiZeroPhysicalPage(
  92     IN PFN_NUMBER PageFrameIndex
  93 );
  94
  95 NTSTATUS
  96 NTAPI
  97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  98 {
  99    PMM_ALLOCATION_REQUEST Request;
 100    PLIST_ENTRY Entry;
 101    KIRQL OldIrql;
 102
 103    if (Page == 0)
 104    {
 105       DPRINT1("Tried to release page zero.\n");
 106       KeBugCheck(MEMORY_MANAGEMENT);
 107    }
 108
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 112       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 113       if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 114       {
 115          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 116          MmDereferencePage(Page);
 117          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 118       }
 119       else
 120       {
 121          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 122          MiZeroPhysicalPage(Page);
 123          Request->Page = Page;
 124          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 125       }
 126    }
 127    else
 128    {
 129       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 130       MmDereferencePage(Page);
 131       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 132    }
 133
 134    return(STATUS_SUCCESS);
 135 }
 136
 137 ULONG
 138 NTAPI
 139 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 140 {
 141     LONG Target = InitialTarget;
 142     ULONG NrFreedPages = 0;
 143     NTSTATUS Status;
 144
 145     /* Make sure we can trim this consumer */
 146     if (!MiMemoryConsumers[Consumer].Trim)
 147     {
 148         /* Return the unmodified initial target */
 149         return InitialTarget;
 150     }
 151
 152     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 153     {
 154         /* Consumer page limit exceeded */
 155         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 156     }
 157     if (MmAvailablePages < MiMinimumAvailablePages)
 158     {
 159         /* Global page limit exceeded */
 160         Target = max(Target, MiMinimumAvailablePages - MmAvailablePages);
 161     }
 162
 163     if (Target)
 164     {
 165         if (!InitialTarget)
 166         {
 167             /* If there was no initial target,
 168              * swap at least MiMinimumPagesPerRun */
 169             Target = max(Target, MiMinimumPagesPerRun);
 170         }
 171
 172         /* Now swap the pages out */
 173         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 174
 175         DPRINT("Trimming consumer %d: Freed %d pages with a target of %d pages\n", Consumer, NrFreedPages, Target);
 176
 177         if (!NT_SUCCESS(Status))
 178         {
 179             KeBugCheck(MEMORY_MANAGEMENT);
 180         }
 181
 182         /* Update the target */
 183         if (NrFreedPages < Target)
 184             Target -= NrFreedPages;
 185         else
 186             Target = 0;
 187
 188         /* Return the remaining pages needed to meet the target */
 189         return Target;
 190     }
 191     else
 192     {
 193         /* Initial target is zero and we don't have anything else to add */
 194         return 0;
 195     }
 196 }
 197
 198 NTSTATUS
 199 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 200 {
 201     PFN_NUMBER CurrentPage;
 202     PFN_NUMBER NextPage;
 203     NTSTATUS Status;
 204
 205     (*NrFreedPages) = 0;
 206
 207     CurrentPage = MmGetLRUFirstUserPage();
 208     while (CurrentPage != 0 && Target > 0)
 209     {
 210         Status = MmPageOutPhysicalAddress(CurrentPage);
 211         if (NT_SUCCESS(Status))
 212         {
 213             DPRINT("Succeeded\n");
 214             Target--;
 215             (*NrFreedPages)++;
 216         }
 217
 218         NextPage = MmGetLRUNextUserPage(CurrentPage);
 219         if (NextPage <= CurrentPage)
 220         {
 221             /* We wrapped around, so we're done */
 222             break;
 223         }
 224         CurrentPage = NextPage;
 225     }
 226
 227     return STATUS_SUCCESS;
 228 }
 229
 230 static BOOLEAN
 231 MiIsBalancerThread(VOID)
 232 {
 233    return (MiBalancerThreadHandle != NULL) &&
 234           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 235 }
 236
 237 VOID
 238 NTAPI
 239 MiDeletePte(IN PMMPTE PointerPte,
 240             IN PVOID VirtualAddress,
 241             IN PEPROCESS CurrentProcess,
 242             IN PMMPTE PrototypePte);
 243
 244 VOID
 245 NTAPI
 246 MmRebalanceMemoryConsumers(VOID)
 247 {
 248 #if (_MI_PAGING_LEVELS == 2)
 249     if(!MiIsBalancerThread())
 250     {
 251         /* Clean up the unused PDEs */
 252         ULONG_PTR Address;
 253         PEPROCESS Process = PsGetCurrentProcess();
 254
 255         /* Acquire PFN lock */
 256         KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 257         PMMPDE pointerPde;
 258         for(Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 259             Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 260             Address += (PAGE_SIZE * PTE_COUNT))
 261         {
 262             if(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] == 0)
 263             {
 264                 pointerPde = MiAddressToPde(Address);
 265                 if(pointerPde->u.Hard.Valid)
 266                     MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 267                 ASSERT(pointerPde->u.Hard.Valid == 0);
 268             }
 269         }
 270         /* Release lock */
 271         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 272     }
 273 #endif
 274
 275     if (MiBalancerThreadHandle != NULL &&
 276         !MiIsBalancerThread())
 277     {
 278         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 279     }
 280 }
 281
 282 NTSTATUS
 283 NTAPI
 284 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 285                             PPFN_NUMBER AllocatedPage)
 286 {
 287    ULONG PagesUsed;
 288    PFN_NUMBER Page;
 289    KIRQL OldIrql;
 290
 291    /*
 292     * Make sure we don't exceed our individual target.
 293     */
 294    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 295    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 296        !MiIsBalancerThread())
 297    {
 298       MmRebalanceMemoryConsumers();
 299    }
 300
 301    /*
 302     * Allocate always memory for the non paged pool and for the pager thread.
 303     */
 304    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 305    {
 306       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 307       Page = MmAllocPage(Consumer);
 308       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 309       if (Page == 0)
 310       {
 311          KeBugCheck(NO_PAGES_AVAILABLE);
 312       }
 313       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 314       *AllocatedPage = Page;
 315       if (MmAvailablePages < MiMinimumAvailablePages)
 316           MmRebalanceMemoryConsumers();
 317       return(STATUS_SUCCESS);
 318    }
 319
 320    /*
 321     * Make sure we don't exceed global targets.
 322     */
 323    if (MmAvailablePages < MiMinimumAvailablePages)
 324    {
 325       MM_ALLOCATION_REQUEST Request;
 326
 327       if (!CanWait)
 328       {
 329          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 330          MmRebalanceMemoryConsumers();
 331          return(STATUS_NO_MEMORY);
 332       }
 333
 334       /* Insert an allocation request. */
 335       Request.Page = 0;
 336       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 337
 338       ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 339       MmRebalanceMemoryConsumers();
 340
 341       KeWaitForSingleObject(&Request.Event,
 342                             0,
 343                             KernelMode,
 344                             FALSE,
 345                             NULL);
 346
 347       Page = Request.Page;
 348       if (Page == 0)
 349       {
 350          KeBugCheck(NO_PAGES_AVAILABLE);
 351       }
 352
 353       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 354       *AllocatedPage = Page;
 355
 356       if (MmAvailablePages < MiMinimumAvailablePages)
 357       {
 358           MmRebalanceMemoryConsumers();
 359       }
 360
 361       return(STATUS_SUCCESS);
 362    }
 363
 364    /*
 365     * Actually allocate the page.
 366     */
 367    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 368    Page = MmAllocPage(Consumer);
 369    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 370    if (Page == 0)
 371    {
 372       KeBugCheck(NO_PAGES_AVAILABLE);
 373    }
 374    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 375    *AllocatedPage = Page;
 376
 377    if (MmAvailablePages < MiMinimumAvailablePages)
 378    {
 379        MmRebalanceMemoryConsumers();
 380    }
 381
 382    return(STATUS_SUCCESS);
 383 }
 384
 385 VOID NTAPI
 386 MiBalancerThread(PVOID Unused)
 387 {
 388    PVOID WaitObjects[2];
 389    NTSTATUS Status;
 390    ULONG i;
 391
 392    WaitObjects[0] = &MiBalancerEvent;
 393    WaitObjects[1] = &MiBalancerTimer;
 394
 395    while (1)
 396    {
 397       Status = KeWaitForMultipleObjects(2,
 398                                         WaitObjects,
 399                                         WaitAny,
 400                                         Executive,
 401                                         KernelMode,
 402                                         FALSE,
 403                                         NULL,
 404                                         NULL);
 405
 406       if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 407       {
 408           ULONG InitialTarget = 0;
 409
 410           do
 411           {
 412               ULONG OldTarget = InitialTarget;
 413
 414               /* Trim each consumer */
 415               for (i = 0; i < MC_MAXIMUM; i++)
 416               {
 417                   InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 418               }
 419
 420               /* No pages left to swap! */
 421               if (InitialTarget != 0 &&
 422                   InitialTarget == OldTarget)
 423               {
 424                   /* Game over */
 425                   KeBugCheck(NO_PAGES_AVAILABLE);
 426               }
 427           } while (InitialTarget != 0);
 428       }
 429       else
 430       {
 431          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 432          KeBugCheck(MEMORY_MANAGEMENT);
 433       }
 434    }
 435 }
 436
 437 VOID
 438 INIT_FUNCTION
 439 NTAPI
 440 MiInitBalancerThread(VOID)
 441 {
 442    KPRIORITY Priority;
 443    NTSTATUS Status;
 444 #if !defined(__GNUC__)
 445
 446    LARGE_INTEGER dummyJunkNeeded;
 447    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 448    ;
 449 #endif
 450
 451
 452    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 453    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 454    KeSetTimerEx(&MiBalancerTimer,
 455 #if defined(__GNUC__)
 456                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 457 #else
 458                 dummyJunkNeeded,
 459 #endif
 460                 2000,         /* 2 sec */
 461                 NULL);
 462
 463    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 464                                  THREAD_ALL_ACCESS,
 465                                  NULL,
 466                                  NULL,
 467                                  &MiBalancerThreadId,
 468                                  (PKSTART_ROUTINE) MiBalancerThread,
 469                                  NULL);
 470    if (!NT_SUCCESS(Status))
 471    {
 472       KeBugCheck(MEMORY_MANAGEMENT);
 473    }
 474
 475    Priority = LOW_REALTIME_PRIORITY + 1;
 476    NtSetInformationThread(MiBalancerThreadHandle,
 477                           ThreadPriority,
 478                           &Priority,
 479                           sizeof(Priority));
 480
 481 }
 482
 483
 484 /* EOF */