reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_NUMBER Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 64;
  63     if ((NrAvailablePages + NrSystemPages) >= 8192)
  64     {
  65         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  66     }
  67     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  68     {
  69         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  70     }
  71     else
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  74     }
  75    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  76 }
  77
  78 VOID
  79 INIT_FUNCTION
  80 NTAPI
  81 MmInitializeMemoryConsumer(ULONG Consumer,
  82                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  83                                             PULONG NrFreed))
  84 {
  85    MiMemoryConsumers[Consumer].Trim = Trim;
  86 }
  87
  88 VOID
  89 NTAPI
  90 MiZeroPhysicalPage(
  91     IN PFN_NUMBER PageFrameIndex
  92 );
  93
  94 NTSTATUS
  95 NTAPI
  96 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  97 {
  98    PMM_ALLOCATION_REQUEST Request;
  99    PLIST_ENTRY Entry;
 100    KIRQL OldIrql;
 101
 102    if (Page == 0)
 103    {
 104       DPRINT1("Tried to release page zero.\n");
 105       KeBugCheck(MEMORY_MANAGEMENT);
 106    }
 107
 108    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 113       {
 114          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 115          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 116          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 117          MmDereferencePage(Page);
 118          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 119       }
 120       else
 121       {
 122          Entry = RemoveHeadList(&AllocationListHead);
 123          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 124          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 125          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 126          MiZeroPhysicalPage(Page);
 127          Request->Page = Page;
 128          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 129       }
 130    }
 131    else
 132    {
 133       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 134       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 135       MmDereferencePage(Page);
 136       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 137    }
 138
 139    return(STATUS_SUCCESS);
 140 }
 141
 142 VOID
 143 NTAPI
 144 MiTrimMemoryConsumer(ULONG Consumer)
 145 {
 146    LONG Target;
 147    ULONG NrFreedPages;
 148
 149    Target = MiMemoryConsumers[Consumer].PagesUsed -
 150             MiMemoryConsumers[Consumer].PagesTarget;
 151    if (Target < 1)
 152    {
 153       Target = 1;
 154    }
 155
 156    if (MiMemoryConsumers[Consumer].Trim != NULL)
 157    {
 158       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 159    }
 160 }
 161
 162 NTSTATUS
 163 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 164 {
 165     PFN_NUMBER CurrentPage;
 166     PFN_NUMBER NextPage;
 167     NTSTATUS Status;
 168
 169     (*NrFreedPages) = 0;
 170
 171     CurrentPage = MmGetLRUFirstUserPage();
 172     while (CurrentPage != 0 && Target > 0)
 173     {
 174         Status = MmPageOutPhysicalAddress(CurrentPage);
 175         if (NT_SUCCESS(Status))
 176         {
 177             DPRINT("Succeeded\n");
 178             Target--;
 179             (*NrFreedPages)++;
 180         }
 181
 182         NextPage = MmGetLRUNextUserPage(CurrentPage);
 183         if (NextPage <= CurrentPage)
 184         {
 185             /* We wrapped around, so we're done */
 186             break;
 187         }
 188         CurrentPage = NextPage;
 189     }
 190
 191     return STATUS_SUCCESS;
 192 }
 193
 194 VOID
 195 NTAPI
 196 MmRebalanceMemoryConsumers(VOID)
 197 {
 198    LONG Target;
 199    ULONG i;
 200    ULONG NrFreedPages;
 201    NTSTATUS Status;
 202
 203    Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 204    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 205
 206    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 207    {
 208       if (MiMemoryConsumers[i].Trim != NULL)
 209       {
 210          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 211          if (!NT_SUCCESS(Status))
 212          {
 213             KeBugCheck(MEMORY_MANAGEMENT);
 214          }
 215          Target = Target - NrFreedPages;
 216       }
 217    }
 218 }
 219
 220 static BOOLEAN
 221 MiIsBalancerThread(VOID)
 222 {
 223    return (MiBalancerThreadHandle != NULL) &&
 224           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 225 }
 226
 227 NTSTATUS
 228 NTAPI
 229 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 230                             PPFN_NUMBER AllocatedPage)
 231 {
 232    ULONG OldUsed;
 233    PFN_NUMBER Page;
 234    KIRQL OldIrql;
 235
 236    /*
 237     * Make sure we don't exceed our individual target.
 238     */
 239    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 240    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 241          !MiIsBalancerThread())
 242    {
 243       if (!CanWait)
 244       {
 245          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 246          return(STATUS_NO_MEMORY);
 247       }
 248       MiTrimMemoryConsumer(Consumer);
 249    }
 250
 251    /*
 252     * Allocate always memory for the non paged pool and for the pager thread.
 253     */
 254    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 255    {
 256       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 257       Page = MmAllocPage(Consumer);
 258       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 259       if (Page == 0)
 260       {
 261          KeBugCheck(NO_PAGES_AVAILABLE);
 262       }
 263       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 264       *AllocatedPage = Page;
 265       if (MmAvailablePages <= MiMinimumAvailablePages &&
 266           MiBalancerThreadHandle != NULL &&
 267           !MiIsBalancerThread())
 268       {
 269          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 270       }
 271       return(STATUS_SUCCESS);
 272    }
 273
 274    /*
 275     * Make sure we don't exceed global targets.
 276     */
 277    if (MmAvailablePages <= MiMinimumAvailablePages)
 278    {
 279       MM_ALLOCATION_REQUEST Request;
 280
 281       if (!CanWait)
 282       {
 283          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 284          return(STATUS_NO_MEMORY);
 285       }
 286
 287       /* Insert an allocation request. */
 288       Request.Page = 0;
 289
 290       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 291       (void)InterlockedIncrementUL(&MiPagesRequired);
 292
 293       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 294
 295       if (MiBalancerThreadHandle != NULL)
 296       {
 297          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 298       }
 299       InsertTailList(&AllocationListHead, &Request.ListEntry);
 300       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 301
 302       KeWaitForSingleObject(&Request.Event,
 303                             0,
 304                             KernelMode,
 305                             FALSE,
 306                             NULL);
 307
 308       Page = Request.Page;
 309       if (Page == 0)
 310       {
 311          KeBugCheck(NO_PAGES_AVAILABLE);
 312       }
 313
 314       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 315       *AllocatedPage = Page;
 316       (void)InterlockedDecrementUL(&MiPagesRequired);
 317
 318       if (MmAvailablePages <= MiMinimumAvailablePages &&
 319           MiBalancerThreadHandle != NULL &&
 320           !MiIsBalancerThread())
 321       {
 322           KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 323       }
 324
 325       return(STATUS_SUCCESS);
 326    }
 327
 328    /*
 329     * Actually allocate the page.
 330     */
 331    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 332    Page = MmAllocPage(Consumer);
 333    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 334    if (Page == 0)
 335    {
 336       KeBugCheck(NO_PAGES_AVAILABLE);
 337    }
 338    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 339    *AllocatedPage = Page;
 340
 341    if (MmAvailablePages <= MiMinimumAvailablePages &&
 342        MiBalancerThreadHandle != NULL &&
 343        !MiIsBalancerThread())
 344    {
 345        KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 346    }
 347
 348    return(STATUS_SUCCESS);
 349 }
 350
 351 VOID NTAPI
 352 MiBalancerThread(PVOID Unused)
 353 {
 354    PVOID WaitObjects[2];
 355    NTSTATUS Status;
 356    ULONG i;
 357    ULONG NrFreedPages;
 358    ULONG NrPagesUsed;
 359    ULONG Target;
 360    BOOLEAN ShouldRun;
 361
 362
 363    WaitObjects[0] = &MiBalancerEvent;
 364    WaitObjects[1] = &MiBalancerTimer;
 365
 366    while (1)
 367    {
 368       Status = KeWaitForMultipleObjects(2,
 369                                         WaitObjects,
 370                                         WaitAny,
 371                                         Executive,
 372                                         KernelMode,
 373                                         FALSE,
 374                                         NULL,
 375                                         NULL);
 376
 377       if (Status == STATUS_SUCCESS)
 378       {
 379          /* MiBalancerEvent */
 380          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 381          {
 382             for (i = 0; i < MC_MAXIMUM; i++)
 383             {
 384                if (MiMemoryConsumers[i].Trim != NULL)
 385                {
 386                   NrFreedPages = 0;
 387                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 388                   if (!NT_SUCCESS(Status))
 389                   {
 390                      KeBugCheck(MEMORY_MANAGEMENT);
 391                   }
 392                }
 393             }
 394          }
 395          InterlockedExchange(&MiBalancerWork, 0);
 396       }
 397       else if (Status == STATUS_SUCCESS + 1)
 398       {
 399          /* MiBalancerTimer */
 400          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 401          for (i = 0; i < MC_MAXIMUM; i++)
 402          {
 403             if (MiMemoryConsumers[i].Trim != NULL)
 404             {
 405                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 406                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 407                {
 408                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 409                   {
 410                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 411                                    MiMinimumPagesPerRun);
 412                   }
 413                   else
 414                   {
 415                      Target = MiMinimumPagesPerRun;
 416                   }
 417                   NrFreedPages = 0;
 418                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 419                   if (!NT_SUCCESS(Status))
 420                   {
 421                      KeBugCheck(MEMORY_MANAGEMENT);
 422                   }
 423                }
 424             }
 425          }
 426       }
 427       else
 428       {
 429          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 430          KeBugCheck(MEMORY_MANAGEMENT);
 431       }
 432    }
 433 }
 434
 435 VOID
 436 INIT_FUNCTION
 437 NTAPI
 438 MiInitBalancerThread(VOID)
 439 {
 440    KPRIORITY Priority;
 441    NTSTATUS Status;
 442 #if !defined(__GNUC__)
 443
 444    LARGE_INTEGER dummyJunkNeeded;
 445    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 446    ;
 447 #endif
 448
 449
 450    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 451    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 452    KeSetTimerEx(&MiBalancerTimer,
 453 #if defined(__GNUC__)
 454                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 455 #else
 456                 dummyJunkNeeded,
 457 #endif
 458                 2000,         /* 2 sec */
 459                 NULL);
 460
 461    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 462                                  THREAD_ALL_ACCESS,
 463                                  NULL,
 464                                  NULL,
 465                                  &MiBalancerThreadId,
 466                                  (PKSTART_ROUTINE) MiBalancerThread,
 467                                  NULL);
 468    if (!NT_SUCCESS(Status))
 469    {
 470       KeBugCheck(MEMORY_MANAGEMENT);
 471    }
 472
 473    Priority = LOW_REALTIME_PRIORITY + 1;
 474    NtSetInformationThread(MiBalancerThreadHandle,
 475                           ThreadPriority,
 476                           &Priority,
 477                           sizeof(Priority));
 478
 479 }
 480
 481
 482 /* EOF */