ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #if defined (ALLOC_PRAGMA)
  18 #pragma alloc_text(INIT, MmInitializeBalancer)
  19 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  20 #pragma alloc_text(INIT, MiInitBalancerThread)
  21 #endif
  22
  23
  24 /* TYPES ********************************************************************/
  25 typedef struct _MM_ALLOCATION_REQUEST
  26 {
  27    PFN_NUMBER Page;
  28    LIST_ENTRY ListEntry;
  29    KEVENT Event;
  30 }
  31 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiMinimumPagesPerRun;
  40
  41 static CLIENT_ID MiBalancerThreadId;
  42 static HANDLE MiBalancerThreadHandle = NULL;
  43 static KEVENT MiBalancerEvent;
  44 static KTIMER MiBalancerTimer;
  45
  46 /* FUNCTIONS ****************************************************************/
  47
  48 VOID
  49 INIT_FUNCTION
  50 NTAPI
  51 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  52 {
  53    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  54    InitializeListHead(&AllocationListHead);
  55    KeInitializeSpinLock(&AllocationListLock);
  56
  57    MiNrTotalPages = NrAvailablePages;
  58
  59    /* Set up targets. */
  60    MiMinimumAvailablePages = 128;
  61    MiMinimumPagesPerRun = 256;
  62     if ((NrAvailablePages + NrSystemPages) >= 8192)
  63     {
  64         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  65     }
  66     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  67     {
  68         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  69     }
  70     else
  71     {
  72         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  73     }
  74    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  75 }
  76
  77 VOID
  78 INIT_FUNCTION
  79 NTAPI
  80 MmInitializeMemoryConsumer(ULONG Consumer,
  81                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  82                                             PULONG NrFreed))
  83 {
  84    MiMemoryConsumers[Consumer].Trim = Trim;
  85 }
  86
  87 VOID
  88 NTAPI
  89 MiZeroPhysicalPage(
  90     IN PFN_NUMBER PageFrameIndex
  91 );
  92
  93 NTSTATUS
  94 NTAPI
  95 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  96 {
  97    PMM_ALLOCATION_REQUEST Request;
  98    PLIST_ENTRY Entry;
  99    KIRQL OldIrql;
 100
 101    if (Page == 0)
 102    {
 103       DPRINT1("Tried to release page zero.\n");
 104       KeBugCheck(MEMORY_MANAGEMENT);
 105    }
 106
 107    if (MmGetReferenceCountPage(Page) == 1)
 108    {
 109       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 110       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 111       if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 112       {
 113          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 114          MmDereferencePage(Page);
 115          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 116       }
 117       else
 118       {
 119          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 120          MiZeroPhysicalPage(Page);
 121          Request->Page = Page;
 122          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 123       }
 124    }
 125    else
 126    {
 127       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 128       MmDereferencePage(Page);
 129       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 130    }
 131
 132    return(STATUS_SUCCESS);
 133 }
 134
 135 ULONG
 136 NTAPI
 137 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 138 {
 139     LONG Target = InitialTarget;
 140     ULONG NrFreedPages = 0;
 141     NTSTATUS Status;
 142
 143     /* Make sure we can trim this consumer */
 144     if (!MiMemoryConsumers[Consumer].Trim)
 145     {
 146         /* Return the unmodified initial target */
 147         return InitialTarget;
 148     }
 149
 150     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 151     {
 152         /* Consumer page limit exceeded */
 153         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 154     }
 155     if (MmAvailablePages < MiMinimumAvailablePages)
 156     {
 157         /* Global page limit exceeded */
 158         Target = max(Target, MiMinimumAvailablePages - MmAvailablePages);
 159     }
 160
 161     if (Target)
 162     {
 163         if (!InitialTarget)
 164         {
 165             /* If there was no initial target,
 166              * swap at least MiMinimumPagesPerRun */
 167             Target = max(Target, MiMinimumPagesPerRun);
 168         }
 169
 170         /* Now swap the pages out */
 171         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 172
 173         DPRINT("Trimming consumer %d: Freed %d pages with a target of %d pages\n", Consumer, NrFreedPages, Target);
 174
 175         if (!NT_SUCCESS(Status))
 176         {
 177             KeBugCheck(MEMORY_MANAGEMENT);
 178         }
 179
 180         /* Update the target */
 181         if (NrFreedPages < Target)
 182             Target -= NrFreedPages;
 183         else
 184             Target = 0;
 185
 186         /* Return the remaining pages needed to meet the target */
 187         return Target;
 188     }
 189     else
 190     {
 191         /* Initial target is zero and we don't have anything else to add */
 192         return 0;
 193     }
 194 }
 195
 196 NTSTATUS
 197 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 198 {
 199     PFN_NUMBER CurrentPage;
 200     PFN_NUMBER NextPage;
 201     NTSTATUS Status;
 202
 203     (*NrFreedPages) = 0;
 204
 205     CurrentPage = MmGetLRUFirstUserPage();
 206     while (CurrentPage != 0 && Target > 0)
 207     {
 208         Status = MmPageOutPhysicalAddress(CurrentPage);
 209         if (NT_SUCCESS(Status))
 210         {
 211             DPRINT("Succeeded\n");
 212             Target--;
 213             (*NrFreedPages)++;
 214         }
 215
 216         NextPage = MmGetLRUNextUserPage(CurrentPage);
 217         if (NextPage <= CurrentPage)
 218         {
 219             /* We wrapped around, so we're done */
 220             break;
 221         }
 222         CurrentPage = NextPage;
 223     }
 224
 225     return STATUS_SUCCESS;
 226 }
 227
 228 static BOOLEAN
 229 MiIsBalancerThread(VOID)
 230 {
 231    return (MiBalancerThreadHandle != NULL) &&
 232           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 233 }
 234
 235 VOID
 236 NTAPI
 237 MmRebalanceMemoryConsumers(VOID)
 238 {
 239     if (MiBalancerThreadHandle != NULL &&
 240         !MiIsBalancerThread())
 241     {
 242         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 243     }
 244 }
 245
 246 NTSTATUS
 247 NTAPI
 248 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 249                             PPFN_NUMBER AllocatedPage)
 250 {
 251    ULONG PagesUsed;
 252    PFN_NUMBER Page;
 253    KIRQL OldIrql;
 254
 255    /*
 256     * Make sure we don't exceed our individual target.
 257     */
 258    PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 259    if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 260        !MiIsBalancerThread())
 261    {
 262       MmRebalanceMemoryConsumers();
 263    }
 264
 265    /*
 266     * Allocate always memory for the non paged pool and for the pager thread.
 267     */
 268    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 269    {
 270       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 271       Page = MmAllocPage(Consumer);
 272       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 273       if (Page == 0)
 274       {
 275          KeBugCheck(NO_PAGES_AVAILABLE);
 276       }
 277       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 278       *AllocatedPage = Page;
 279       if (MmAvailablePages < MiMinimumAvailablePages)
 280           MmRebalanceMemoryConsumers();
 281       return(STATUS_SUCCESS);
 282    }
 283
 284    /*
 285     * Make sure we don't exceed global targets.
 286     */
 287    if (MmAvailablePages < MiMinimumAvailablePages)
 288    {
 289       MM_ALLOCATION_REQUEST Request;
 290
 291       if (!CanWait)
 292       {
 293          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 294          MmRebalanceMemoryConsumers();
 295          return(STATUS_NO_MEMORY);
 296       }
 297
 298       /* Insert an allocation request. */
 299       Request.Page = 0;
 300       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 301
 302       ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 303       MmRebalanceMemoryConsumers();
 304
 305       KeWaitForSingleObject(&Request.Event,
 306                             0,
 307                             KernelMode,
 308                             FALSE,
 309                             NULL);
 310
 311       Page = Request.Page;
 312       if (Page == 0)
 313       {
 314          KeBugCheck(NO_PAGES_AVAILABLE);
 315       }
 316
 317       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 318       *AllocatedPage = Page;
 319
 320       if (MmAvailablePages < MiMinimumAvailablePages)
 321       {
 322           MmRebalanceMemoryConsumers();
 323       }
 324
 325       return(STATUS_SUCCESS);
 326    }
 327
 328    /*
 329     * Actually allocate the page.
 330     */
 331    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 332    Page = MmAllocPage(Consumer);
 333    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 334    if (Page == 0)
 335    {
 336       KeBugCheck(NO_PAGES_AVAILABLE);
 337    }
 338    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 339    *AllocatedPage = Page;
 340
 341    if (MmAvailablePages < MiMinimumAvailablePages)
 342    {
 343        MmRebalanceMemoryConsumers();
 344    }
 345
 346    return(STATUS_SUCCESS);
 347 }
 348
 349 VOID NTAPI
 350 MiBalancerThread(PVOID Unused)
 351 {
 352    PVOID WaitObjects[2];
 353    NTSTATUS Status;
 354    ULONG i;
 355
 356    WaitObjects[0] = &MiBalancerEvent;
 357    WaitObjects[1] = &MiBalancerTimer;
 358
 359    while (1)
 360    {
 361       Status = KeWaitForMultipleObjects(2,
 362                                         WaitObjects,
 363                                         WaitAny,
 364                                         Executive,
 365                                         KernelMode,
 366                                         FALSE,
 367                                         NULL,
 368                                         NULL);
 369
 370       if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 371       {
 372           ULONG InitialTarget = 0;
 373
 374           do
 375           {
 376               ULONG OldTarget = InitialTarget;
 377
 378               /* Trim each consumer */
 379               for (i = 0; i < MC_MAXIMUM; i++)
 380               {
 381                   InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 382               }
 383
 384               /* No pages left to swap! */
 385               if (InitialTarget != 0 &&
 386                   InitialTarget == OldTarget)
 387               {
 388                   /* Game over */
 389                   KeBugCheck(NO_PAGES_AVAILABLE);
 390               }
 391           } while (InitialTarget != 0);
 392       }
 393       else
 394       {
 395          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 396          KeBugCheck(MEMORY_MANAGEMENT);
 397       }
 398    }
 399 }
 400
 401 VOID
 402 INIT_FUNCTION
 403 NTAPI
 404 MiInitBalancerThread(VOID)
 405 {
 406    KPRIORITY Priority;
 407    NTSTATUS Status;
 408 #if !defined(__GNUC__)
 409
 410    LARGE_INTEGER dummyJunkNeeded;
 411    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 412    ;
 413 #endif
 414
 415
 416    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 417    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 418    KeSetTimerEx(&MiBalancerTimer,
 419 #if defined(__GNUC__)
 420                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 421 #else
 422                 dummyJunkNeeded,
 423 #endif
 424                 2000,         /* 2 sec */
 425                 NULL);
 426
 427    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 428                                  THREAD_ALL_ACCESS,
 429                                  NULL,
 430                                  NULL,
 431                                  &MiBalancerThreadId,
 432                                  (PKSTART_ROUTINE) MiBalancerThread,
 433                                  NULL);
 434    if (!NT_SUCCESS(Status))
 435    {
 436       KeBugCheck(MEMORY_MANAGEMENT);
 437    }
 438
 439    Priority = LOW_REALTIME_PRIORITY + 1;
 440    NtSetInformationThread(MiBalancerThreadHandle,
 441                           ThreadPriority,
 442                           &Priority,
 443                           sizeof(Priority));
 444
 445 }
 446
 447
 448 /* EOF */