ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_TYPE Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID MmPrintMemoryStatistic(VOID)
  51 {
  52    DbgPrint("MC_CACHE %d, MC_USER %d, MC_PPOOL %d, MC_NPPOOL %d, MmAvailablePages %d\n",
  53             MiMemoryConsumers[MC_CACHE].PagesUsed, MiMemoryConsumers[MC_USER].PagesUsed,
  54             MiMemoryConsumers[MC_PPOOL].PagesUsed, MiMemoryConsumers[MC_NPPOOL].PagesUsed,
  55             MmAvailablePages);
  56 }
  57
  58 VOID
  59 INIT_FUNCTION
  60 NTAPI
  61 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  62 {
  63    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  64    InitializeListHead(&AllocationListHead);
  65    KeInitializeSpinLock(&AllocationListLock);
  66
  67    MiNrTotalPages = NrAvailablePages;
  68
  69    /* Set up targets. */
  70    MiMinimumAvailablePages = 64;
  71     if ((NrAvailablePages + NrSystemPages) >= 8192)
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  74     }
  75     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  76     {
  77         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  78     }
  79     else
  80     {
  81         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  82     }
  83    MiMemoryConsumers[MC_USER].PagesTarget =
  84       NrAvailablePages - MiMinimumAvailablePages;
  85    MiMemoryConsumers[MC_PPOOL].PagesTarget = NrAvailablePages / 2;
  86    MiMemoryConsumers[MC_NPPOOL].PagesTarget = 0xFFFFFFFF;
  87    MiMemoryConsumers[MC_NPPOOL].PagesUsed = NrSystemPages;
  88    MiMemoryConsumers[MC_SYSTEM].PagesTarget = 0xFFFFFFFF;
  89    MiMemoryConsumers[MC_SYSTEM].PagesUsed = 0;
  90 }
  91
  92 VOID
  93 INIT_FUNCTION
  94 NTAPI
  95 MmInitializeMemoryConsumer(ULONG Consumer,
  96                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  97                                             PULONG NrFreed))
  98 {
  99    MiMemoryConsumers[Consumer].Trim = Trim;
 100 }
 101
 102 NTSTATUS
 103 NTAPI
 104 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_TYPE Page)
 105 {
 106    PMM_ALLOCATION_REQUEST Request;
 107    PLIST_ENTRY Entry;
 108    KIRQL OldIrql;
 109
 110    if (Page == 0)
 111    {
 112       DPRINT1("Tried to release page zero.\n");
 113       KeBugCheck(MEMORY_MANAGEMENT);
 114    }
 115
 116    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 117    if (MmGetReferenceCountPage(Page) == 1)
 118    {
 119       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 120       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 121       {
 122          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 123          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 124          MmDereferencePage(Page);
 125          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 126       }
 127       else
 128       {
 129          Entry = RemoveHeadList(&AllocationListHead);
 130          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 131          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 132          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 133          MiZeroPage(Page);
 134          Request->Page = Page;
 135          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 136       }
 137    }
 138    else
 139    {
 140       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 141       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 142       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 143       MmDereferencePage(Page);
 144       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 145    }
 146
 147    return(STATUS_SUCCESS);
 148 }
 149
 150 VOID
 151 NTAPI
 152 MiTrimMemoryConsumer(ULONG Consumer)
 153 {
 154    LONG Target;
 155    ULONG NrFreedPages;
 156
 157    Target = MiMemoryConsumers[Consumer].PagesUsed -
 158             MiMemoryConsumers[Consumer].PagesTarget;
 159    if (Target < 1)
 160    {
 161       Target = 1;
 162    }
 163
 164    if (MiMemoryConsumers[Consumer].Trim != NULL)
 165    {
 166       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 167    }
 168 }
 169
 170 NTSTATUS
 171 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 172 {
 173     PFN_TYPE CurrentPage;
 174     PFN_TYPE NextPage;
 175     NTSTATUS Status;
 176
 177     (*NrFreedPages) = 0;
 178
 179     CurrentPage = MmGetLRUFirstUserPage();
 180     while (CurrentPage != 0 && Target > 0)
 181     {
 182         NextPage = MmGetLRUNextUserPage(CurrentPage);
 183
 184         Status = MmPageOutPhysicalAddress(CurrentPage);
 185         if (NT_SUCCESS(Status))
 186         {
 187             DPRINT("Succeeded\n");
 188             Target--;
 189             (*NrFreedPages)++;
 190         }
 191         else if (Status == STATUS_PAGEFILE_QUOTA)
 192         {
 193             MmRemoveLRUUserPage(CurrentPage);
 194             MmInsertLRULastUserPage(CurrentPage);
 195         }
 196
 197         CurrentPage = NextPage;
 198     }
 199     return(STATUS_SUCCESS);
 200 }
 201
 202 VOID
 203 NTAPI
 204 MmRebalanceMemoryConsumers(VOID)
 205 {
 206    LONG Target;
 207    ULONG i;
 208    ULONG NrFreedPages;
 209    NTSTATUS Status;
 210
 211    Target = (MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 212    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 213
 214    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 215    {
 216       if (MiMemoryConsumers[i].Trim != NULL)
 217       {
 218          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 219          if (!NT_SUCCESS(Status))
 220          {
 221             KeBugCheck(MEMORY_MANAGEMENT);
 222          }
 223          Target = Target - NrFreedPages;
 224       }
 225    }
 226 }
 227
 228 static BOOLEAN
 229 MiIsBalancerThread(VOID)
 230 {
 231    return MiBalancerThreadHandle != NULL &&
 232           PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
 233 }
 234
 235 NTSTATUS
 236 NTAPI
 237 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 238                             PPFN_TYPE AllocatedPage)
 239 {
 240    ULONG OldUsed;
 241    PFN_TYPE Page;
 242    KIRQL OldIrql;
 243
 244    /*
 245     * Make sure we don't exceed our individual target.
 246     */
 247    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 248    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 249          !MiIsBalancerThread())
 250    {
 251       if (!CanWait)
 252       {
 253          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 254          return(STATUS_NO_MEMORY);
 255       }
 256       MiTrimMemoryConsumer(Consumer);
 257    }
 258
 259    /*
 260     * Allocate always memory for the non paged pool and for the pager thread.
 261     */
 262    if ((Consumer == MC_NPPOOL) || (Consumer == MC_SYSTEM) || MiIsBalancerThread())
 263    {
 264       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 265       Page = MmAllocPage(Consumer, 0);
 266       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 267       if (Page == 0)
 268       {
 269          KeBugCheck(NO_PAGES_AVAILABLE);
 270       }
 271       *AllocatedPage = Page;
 272       if (MmAvailablePages <= MiMinimumAvailablePages &&
 273             MiBalancerThreadHandle != NULL)
 274       {
 275          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 276       }
 277       return(STATUS_SUCCESS);
 278    }
 279
 280    /*
 281     * Make sure we don't exceed global targets.
 282     */
 283    if (MmAvailablePages <= MiMinimumAvailablePages)
 284    {
 285       MM_ALLOCATION_REQUEST Request;
 286
 287       if (!CanWait)
 288       {
 289          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 290          return(STATUS_NO_MEMORY);
 291       }
 292
 293       /* Insert an allocation request. */
 294       Request.Page = 0;
 295
 296       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 297       (void)InterlockedIncrementUL(&MiPagesRequired);
 298
 299       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 300
 301       if (MiBalancerThreadHandle != NULL)
 302       {
 303          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 304       }
 305       InsertTailList(&AllocationListHead, &Request.ListEntry);
 306       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 307
 308       KeWaitForSingleObject(&Request.Event,
 309                             0,
 310                             KernelMode,
 311                             FALSE,
 312                             NULL);
 313
 314       Page = Request.Page;
 315       if (Page == 0)
 316       {
 317          KeBugCheck(NO_PAGES_AVAILABLE);
 318       }
 319       /* Update the Consumer */
 320       MiGetPfnEntry(Page)->u3.e1.PageLocation = Consumer;
 321       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 322       *AllocatedPage = Page;
 323       (void)InterlockedDecrementUL(&MiPagesRequired);
 324       return(STATUS_SUCCESS);
 325    }
 326
 327    /*
 328     * Actually allocate the page.
 329     */
 330    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 331    Page = MmAllocPage(Consumer, 0);
 332    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 333    if (Page == 0)
 334    {
 335       KeBugCheck(NO_PAGES_AVAILABLE);
 336    }
 337    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 338    *AllocatedPage = Page;
 339
 340    return(STATUS_SUCCESS);
 341 }
 342
 343 VOID NTAPI
 344 MiBalancerThread(PVOID Unused)
 345 {
 346    PVOID WaitObjects[2];
 347    NTSTATUS Status;
 348    ULONG i;
 349    ULONG NrFreedPages;
 350    ULONG NrPagesUsed;
 351    ULONG Target;
 352    BOOLEAN ShouldRun;
 353
 354
 355    WaitObjects[0] = &MiBalancerEvent;
 356    WaitObjects[1] = &MiBalancerTimer;
 357
 358    while (1)
 359    {
 360       Status = KeWaitForMultipleObjects(2,
 361                                         WaitObjects,
 362                                         WaitAny,
 363                                         Executive,
 364                                         KernelMode,
 365                                         FALSE,
 366                                         NULL,
 367                                         NULL);
 368
 369       if (Status == STATUS_SUCCESS)
 370       {
 371          /* MiBalancerEvent */
 372          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 373          {
 374             for (i = 0; i < MC_MAXIMUM; i++)
 375             {
 376                if (MiMemoryConsumers[i].Trim != NULL)
 377                {
 378                   NrFreedPages = 0;
 379                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 380                   if (!NT_SUCCESS(Status))
 381                   {
 382                      KeBugCheck(MEMORY_MANAGEMENT);
 383                   }
 384                }
 385             }
 386          }
 387          InterlockedExchange(&MiBalancerWork, 0);
 388       }
 389       else if (Status == STATUS_SUCCESS + 1)
 390       {
 391          /* MiBalancerTimer */
 392          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 393          for (i = 0; i < MC_MAXIMUM; i++)
 394          {
 395             if (MiMemoryConsumers[i].Trim != NULL)
 396             {
 397                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 398                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 399                {
 400                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 401                   {
 402                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 403                                    MiMinimumPagesPerRun);
 404                   }
 405                   else
 406                   {
 407                      Target = MiMinimumPagesPerRun;
 408                   }
 409                   NrFreedPages = 0;
 410                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 411                   if (!NT_SUCCESS(Status))
 412                   {
 413                      KeBugCheck(MEMORY_MANAGEMENT);
 414                   }
 415                }
 416             }
 417          }
 418       }
 419       else
 420       {
 421          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 422          KeBugCheck(MEMORY_MANAGEMENT);
 423       }
 424    }
 425 }
 426
 427 VOID
 428 INIT_FUNCTION
 429 NTAPI
 430 MiInitBalancerThread(VOID)
 431 {
 432    KPRIORITY Priority;
 433    NTSTATUS Status;
 434 #if !defined(__GNUC__)
 435
 436    LARGE_INTEGER dummyJunkNeeded;
 437    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 438    ;
 439 #endif
 440
 441
 442    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 443    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 444    KeSetTimerEx(&MiBalancerTimer,
 445 #if defined(__GNUC__)
 446                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 447 #else
 448                 dummyJunkNeeded,
 449 #endif
 450                 2000,         /* 2 sec */
 451                 NULL);
 452
 453    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 454                                  THREAD_ALL_ACCESS,
 455                                  NULL,
 456                                  NULL,
 457                                  &MiBalancerThreadId,
 458                                  (PKSTART_ROUTINE) MiBalancerThread,
 459                                  NULL);
 460    if (!NT_SUCCESS(Status))
 461    {
 462       KeBugCheck(MEMORY_MANAGEMENT);
 463    }
 464
 465    Priority = LOW_REALTIME_PRIORITY + 1;
 466    NtSetInformationThread(MiBalancerThreadHandle,
 467                           ThreadPriority,
 468                           &Priority,
 469                           sizeof(Priority));
 470
 471 }
 472
 473
 474 /* EOF */