reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #if defined (ALLOC_PRAGMA)
  18 #pragma alloc_text(INIT, MmInitializeBalancer)
  19 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  20 #pragma alloc_text(INIT, MiInitBalancerThread)
  21 #endif
  22
  23
  24 /* TYPES ********************************************************************/
  25 typedef struct _MM_ALLOCATION_REQUEST
  26 {
  27    PFN_NUMBER Page;
  28    LIST_ENTRY ListEntry;
  29    KEVENT Event;
  30 }
  31 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiMinimumPagesPerRun;
  40
  41 static CLIENT_ID MiBalancerThreadId;
  42 static HANDLE MiBalancerThreadHandle = NULL;
  43 static KEVENT MiBalancerEvent;
  44 static KTIMER MiBalancerTimer;
  45
  46 /* FUNCTIONS ****************************************************************/
  47
  48 VOID
  49 INIT_FUNCTION
  50 NTAPI
  51 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  52 {
  53    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  54    InitializeListHead(&AllocationListHead);
  55    KeInitializeSpinLock(&AllocationListLock);
  56
  57    MiNrTotalPages = NrAvailablePages;
  58
  59    /* Set up targets. */
  60    MiMinimumAvailablePages = 128;
  61    MiMinimumPagesPerRun = 256;
  62     if ((NrAvailablePages + NrSystemPages) >= 8192)
  63     {
  64         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  65     }
  66     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  67     {
  68         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  69     }
  70     else
  71     {
  72         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  73     }
  74    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  75 }
  76
  77 VOID
  78 INIT_FUNCTION
  79 NTAPI
  80 MmInitializeMemoryConsumer(ULONG Consumer,
  81                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  82                                             PULONG NrFreed))
  83 {
  84    MiMemoryConsumers[Consumer].Trim = Trim;
  85 }
  86
  87 VOID
  88 NTAPI
  89 MiZeroPhysicalPage(
  90     IN PFN_NUMBER PageFrameIndex
  91 );
  92
  93 NTSTATUS
  94 NTAPI
  95 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  96 {
  97    PMM_ALLOCATION_REQUEST Request;
  98    PLIST_ENTRY Entry;
  99    KIRQL OldIrql;
 100
 101    if (Page == 0)
 102    {
 103       DPRINT1("Tried to release page zero.\n");
 104       KeBugCheck(MEMORY_MANAGEMENT);
 105    }
 106
 107    if (MmGetReferenceCountPage(Page) == 1)
 108    {
 109       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 110       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 111       if (MmAvailablePages < MiMinimumAvailablePages ||
 112              (Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 113       {
 114          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 115          MmDereferencePage(Page);
 116          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 117       }
 118       else
 119       {
 120          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 121          MiZeroPhysicalPage(Page);
 122          Request->Page = Page;
 123          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 124       }
 125    }
 126    else
 127    {
 128       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 129       MmDereferencePage(Page);
 130       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 131    }
 132
 133    return(STATUS_SUCCESS);
 134 }
 135
 136 VOID
 137 NTAPI
 138 MiTrimMemoryConsumer(ULONG Consumer)
 139 {
 140     LONG Target = 0;
 141     ULONG NrFreedPages = 0;
 142     NTSTATUS Status;
 143
 144     /* Make sure we can trim this consumer */
 145     if (!MiMemoryConsumers[Consumer].Trim)
 146         return;
 147
 148     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 149     {
 150         /* Consumer page limit exceeded */
 151         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 152     }
 153     if (MmAvailablePages < MiMinimumAvailablePages)
 154     {
 155         /* Global page limit exceeded */
 156         Target = max(Target, MiMinimumAvailablePages - MmAvailablePages);
 157     }
 158
 159     if (Target)
 160     {
 161         /* Swap at least MiMinimumPagesPerRun */
 162         Target = max(Target, MiMinimumPagesPerRun);
 163
 164         /* Now swap the pages out */
 165         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 166
 167         if (!ExpInTextModeSetup)
 168             DPRINT1("Trimming consumer %d: Freed %d pages with a target of %d pages\n", Consumer, NrFreedPages, Target);
 169
 170                 if (NrFreedPages == 0)
 171                     DPRINT1("Ran out of pages to swap! Complete memory exhaustion is imminent!\n");
 172
 173         if (!NT_SUCCESS(Status))
 174         {
 175             KeBugCheck(MEMORY_MANAGEMENT);
 176         }
 177     }
 178 }
 179
 180 NTSTATUS
 181 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 182 {
 183     PFN_NUMBER CurrentPage;
 184     PFN_NUMBER NextPage;
 185     NTSTATUS Status;
 186
 187     (*NrFreedPages) = 0;
 188
 189     CurrentPage = MmGetLRUFirstUserPage();
 190     while (CurrentPage != 0 && Target > 0)
 191     {
 192         Status = MmPageOutPhysicalAddress(CurrentPage);
 193         if (NT_SUCCESS(Status))
 194         {
 195             DPRINT("Succeeded\n");
 196             Target--;
 197             (*NrFreedPages)++;
 198         }
 199
 200         NextPage = MmGetLRUNextUserPage(CurrentPage);
 201         if (NextPage <= CurrentPage)
 202         {
 203             /* We wrapped around, so we're done */
 204             break;
 205         }
 206         CurrentPage = NextPage;
 207     }
 208
 209     return STATUS_SUCCESS;
 210 }
 211
 212 static BOOLEAN
 213 MiIsBalancerThread(VOID)
 214 {
 215    return (MiBalancerThreadHandle != NULL) &&
 216           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 217 }
 218
 219 VOID
 220 NTAPI
 221 MmRebalanceMemoryConsumers(VOID)
 222 {
 223     if (MiBalancerThreadHandle != NULL &&
 224         !MiIsBalancerThread())
 225     {
 226         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 227     }
 228 }
 229
 230 NTSTATUS
 231 NTAPI
 232 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 233                             PPFN_NUMBER AllocatedPage)
 234 {
 235    ULONG PagesUsed;
 236    PFN_NUMBER Page;
 237    KIRQL OldIrql;
 238
 239    /*
 240     * Make sure we don't exceed our individual target.
 241     */
 242    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed) + 1;
 243    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 244        !MiIsBalancerThread())
 245    {
 246       MmRebalanceMemoryConsumers();
 247    }
 248
 249    /*
 250     * Allocate always memory for the non paged pool and for the pager thread.
 251     */
 252    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 253    {
 254       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 255       Page = MmAllocPage(Consumer);
 256       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 257       if (Page == 0)
 258       {
 259          KeBugCheck(NO_PAGES_AVAILABLE);
 260       }
 261       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 262       *AllocatedPage = Page;
 263       if (MmAvailablePages < MiMinimumAvailablePages)
 264           MmRebalanceMemoryConsumers();
 265       return(STATUS_SUCCESS);
 266    }
 267
 268    /*
 269     * Make sure we don't exceed global targets.
 270     */
 271    if (MmAvailablePages < MiMinimumAvailablePages)
 272    {
 273       MM_ALLOCATION_REQUEST Request;
 274
 275       if (!CanWait)
 276       {
 277          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 278          return(STATUS_NO_MEMORY);
 279       }
 280
 281       /* Insert an allocation request. */
 282       Request.Page = 0;
 283       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 284
 285       ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 286       MmRebalanceMemoryConsumers();
 287
 288       KeWaitForSingleObject(&Request.Event,
 289                             0,
 290                             KernelMode,
 291                             FALSE,
 292                             NULL);
 293
 294       Page = Request.Page;
 295       if (Page == 0)
 296       {
 297          KeBugCheck(NO_PAGES_AVAILABLE);
 298       }
 299
 300       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 301       *AllocatedPage = Page;
 302
 303       if (MmAvailablePages < MiMinimumAvailablePages)
 304       {
 305           MmRebalanceMemoryConsumers();
 306       }
 307
 308       return(STATUS_SUCCESS);
 309    }
 310
 311    /*
 312     * Actually allocate the page.
 313     */
 314    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 315    Page = MmAllocPage(Consumer);
 316    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 317    if (Page == 0)
 318    {
 319       KeBugCheck(NO_PAGES_AVAILABLE);
 320    }
 321    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 322    *AllocatedPage = Page;
 323
 324    if (MmAvailablePages < MiMinimumAvailablePages)
 325    {
 326        MmRebalanceMemoryConsumers();
 327    }
 328
 329    return(STATUS_SUCCESS);
 330 }
 331
 332 VOID NTAPI
 333 MiBalancerThread(PVOID Unused)
 334 {
 335    PVOID WaitObjects[2];
 336    NTSTATUS Status;
 337    ULONG i;
 338
 339    WaitObjects[0] = &MiBalancerEvent;
 340    WaitObjects[1] = &MiBalancerTimer;
 341
 342    while (1)
 343    {
 344       Status = KeWaitForMultipleObjects(2,
 345                                         WaitObjects,
 346                                         WaitAny,
 347                                         Executive,
 348                                         KernelMode,
 349                                         FALSE,
 350                                         NULL,
 351                                         NULL);
 352
 353       if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 354       {
 355           for (i = 0; i < MC_MAXIMUM; i++)
 356           {
 357               MiTrimMemoryConsumer(i);
 358           }
 359       }
 360       else
 361       {
 362          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 363          KeBugCheck(MEMORY_MANAGEMENT);
 364       }
 365    }
 366 }
 367
 368 VOID
 369 INIT_FUNCTION
 370 NTAPI
 371 MiInitBalancerThread(VOID)
 372 {
 373    KPRIORITY Priority;
 374    NTSTATUS Status;
 375 #if !defined(__GNUC__)
 376
 377    LARGE_INTEGER dummyJunkNeeded;
 378    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 379    ;
 380 #endif
 381
 382
 383    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 384    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 385    KeSetTimerEx(&MiBalancerTimer,
 386 #if defined(__GNUC__)
 387                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 388 #else
 389                 dummyJunkNeeded,
 390 #endif
 391                 2000,         /* 2 sec */
 392                 NULL);
 393
 394    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 395                                  THREAD_ALL_ACCESS,
 396                                  NULL,
 397                                  NULL,
 398                                  &MiBalancerThreadId,
 399                                  (PKSTART_ROUTINE) MiBalancerThread,
 400                                  NULL);
 401    if (!NT_SUCCESS(Status))
 402    {
 403       KeBugCheck(MEMORY_MANAGEMENT);
 404    }
 405
 406    Priority = LOW_REALTIME_PRIORITY + 1;
 407    NtSetInformationThread(MiBalancerThreadHandle,
 408                           ThreadPriority,
 409                           &Priority,
 410                           sizeof(Priority));
 411
 412 }
 413
 414
 415 /* EOF */